[PATCH 6/6] eb-unbind
Chris Wilson
chris at chris-wilson.co.uk
Fri Mar 13 09:05:27 UTC 2020
---
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 593 ++++++++++++++++--
drivers/gpu/drm/i915/gt/intel_gtt.c | 4 +
drivers/gpu/drm/i915/gt/intel_gtt.h | 2 +
drivers/gpu/drm/i915/gt/intel_ppgtt.c | 3 +-
drivers/gpu/drm/i915/i915_vma.c | 61 +-
drivers/gpu/drm/i915/i915_vma.h | 18 +-
6 files changed, 580 insertions(+), 101 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 2d43b7435724..f860c9188197 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -31,6 +31,9 @@ struct eb_vma {
struct i915_vma *vma;
unsigned int flags;
+ struct drm_mm_node hole;
+ unsigned int bind_flags;
+
/** This vma's place in the execbuf reservation list */
struct drm_i915_gem_exec_object2 *exec;
struct list_head bind_link;
@@ -60,6 +63,12 @@ static struct eb_vma_array *eb_vma_array_create(unsigned int count)
return arr;
}
+static struct eb_vma_array *eb_vma_array_get(struct eb_vma_array *arr)
+{
+ kref_get(&arr->kref);
+ return arr;
+}
+
static void eb_vma_array_destroy(struct kref *kref)
{
struct eb_vma_array *arr = container_of(kref, typeof(*arr), kref);
@@ -83,7 +92,8 @@ enum {
#define __EXEC_OBJECT_HAS_FENCE BIT(30)
#define __EXEC_OBJECT_NEEDS_MAP BIT(29)
#define __EXEC_OBJECT_NEEDS_BIAS BIT(28)
-#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */
+#define __EXEC_OBJECT_HAS_PAGES BIT(27)
+#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27) /* all of the above */
#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
#define __EXEC_HAS_RELOC BIT(31)
@@ -379,6 +389,10 @@ static int eb_create(struct i915_execbuffer *eb)
eb->lut_size = -eb->buffer_count;
}
+ INIT_LIST_HEAD(&eb->relocs);
+ INIT_LIST_HEAD(&eb->unbound);
+ INIT_LIST_HEAD(&eb->lock);
+
return 0;
}
@@ -459,10 +473,15 @@ static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags)
static inline void
eb_unreserve_vma(struct eb_vma *ev)
{
- if (!(ev->flags & __EXEC_OBJECT_HAS_PIN))
- return;
+ if (ev->flags & __EXEC_OBJECT_HAS_PIN)
+ __eb_unreserve_vma(ev->vma, ev->flags);
- __eb_unreserve_vma(ev->vma, ev->flags);
+ if (ev->flags & __EXEC_OBJECT_HAS_PAGES) {
+ i915_vma_put_pages(ev->vma);
+ ev->flags &= ~__EXEC_OBJECT_HAS_PAGES;
+ }
+
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
ev->flags &= ~__EXEC_OBJECT_RESERVED;
}
@@ -525,6 +544,7 @@ eb_add_vma(struct i915_execbuffer *eb,
GEM_BUG_ON(i915_vma_is_closed(vma));
+ memset(&ev->hole, 0, sizeof(ev->hole));
ev->vma = i915_vma_get(vma);
ev->exec = entry;
ev->flags = entry->flags;
@@ -566,6 +586,7 @@ eb_add_vma(struct i915_execbuffer *eb,
} else {
eb_unreserve_vma(ev);
list_add_tail(&ev->bind_link, &eb->unbound);
+ i915_vma_get(vma);
}
list_add_tail(&ev->lock_link, &eb->lock);
@@ -588,55 +609,306 @@ static inline int use_cpu_reloc(const struct reloc_cache *cache,
obj->cache_level != I915_CACHE_NONE);
}
-static int eb_reserve_vma(const struct i915_execbuffer *eb,
- struct eb_vma *ev,
- u64 pin_flags)
+struct eb_vm_work {
+ struct dma_fence_work base;
+ struct list_head unbound;
+ struct eb_vma_array *array;
+ struct i915_address_space *vm;
+ struct list_head eviction_list;
+ u64 *p_flags;
+ u64 active;
+};
+
+static inline u64 node_end(const struct drm_mm_node *node)
+{
+ return node->start + node->size;
+}
+
+static void __evict(struct i915_vma *vma)
+{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ if (i915_vma_is_map_and_fenceable(vma)) {
+ /*
+ * Check that we have flushed all writes through the GGTT
+ * before the unbind, other due to non-strict nature of those
+ * indirect writes they may end up referencing the GGTT PTE
+ * after the unbind.
+ *
+ * Note that we may be concurrently poking at the GGTT_WRITE
+ * bit from set-domain, as we mark all GGTT vma associated
+ * with an object. We know this is for another vma, as we
+ * are currently unbinding this one -- so if this vma will be
+ * reused, it will be refaulted and have its dirty bit set
+ * before the next write.
+ */
+ i915_vma_flush_writes(vma);
+
+ /* release the fence reg _after_ flushing */
+ i915_vma_revoke_fence(vma);
+
+ /* Force a pagefault for domain tracking on next user access */
+ i915_vma_revoke_mmap(vma);
+
+ __i915_vma_iounmap(vma);
+ clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
+ }
+
+ if (likely(atomic_read(&vma->vm->open)))
+ vma->ops->unbind_vma(vma);
+
+ i915_vma_unbind_pages(vma);
+}
+
+static bool evict_for_node(struct eb_vm_work *work,
+ struct eb_vma *const target,
+ unsigned int flags)
+{
+ struct i915_address_space *vm = target->vma->vm;
+ const unsigned long color = target->vma->node.color;
+ const u64 start = target->vma->node.start;
+ const u64 end = start + target->vma->node.size;
+ u64 hole_start = start, hole_end = end;
+ struct drm_mm_node *node;
+ LIST_HEAD(eviction_list);
+ struct i915_vma *vma;
+
+ lockdep_assert_held(&vm->mutex);
+ GEM_BUG_ON(drm_mm_node_allocated(&target->vma->node));
+ GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
+ GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
+
+ if (i915_vm_has_cache_coloring(vm)) {
+ /* Expand search to cover neighbouring guard pages (or lack!) */
+ if (hole_start)
+ hole_start -= I915_GTT_PAGE_SIZE;
+
+ /* Always look at the page afterwards to avoid the end-of-GTT */
+ hole_end += I915_GTT_PAGE_SIZE;
+ }
+ GEM_BUG_ON(hole_start >= hole_end);
+
+ drm_mm_for_each_node_in_range(node, &vm->mm, hole_start, hole_end) {
+ GEM_BUG_ON(node == &target->vma->node);
+
+ /* If we find any non-objects (!vma), we cannot evict them */
+ if (node->color == I915_COLOR_UNEVICTABLE)
+ goto err;
+
+ GEM_BUG_ON(!drm_mm_node_allocated(node));
+ vma = container_of(node, typeof(*vma), node);
+
+ /*
+ * If we are using coloring to insert guard pages between
+ * different cache domains within the address space, we have
+ * to check whether the objects on either side of our range
+ * abutt and conflict. If they are in conflict, then we evict
+ * those as well to make room for our guard pages.
+ */
+ if (i915_vm_has_cache_coloring(vm)) {
+ if (node_end(node) == start && node->color == color)
+ continue;
+
+ if (node->start == end && node->color == color)
+ continue;
+ }
+
+ if (i915_vma_is_pinned(vma))
+ goto err;
+
+ if (flags & PIN_NONBLOCK && i915_vma_is_active(vma))
+ goto err;
+
+ if (i915_sw_fence_await_active(&work->base.chain,
+ &vma->active,
+ I915_ACTIVE_AWAIT_ALL))
+ goto err;
+
+ if (i915_active_acquire(&vma->active))
+ goto err;
+
+ i915_active_set_exclusive(&vma->active, &work->base.dma);
+ list_move(&vma->vm_link, &eviction_list);
+ i915_active_release(&vma->active);
+ }
+
+ /* No overlapping nodes to evict, claim the slot for ourselves! */
+ if (list_empty(&eviction_list))
+ return drm_mm_reserve_node(&vm->mm, &target->vma->node) == 0;
+
+ /*
+ * Mark this range as reserved.
+ *
+ * We have not yet removed the PTEs for the old evicted nodes, so
+ * must prevent this range from being reused for anything else. The
+ * PTE will be cleared when the range is idle (during the rebind
+ * phase in the worker).
+ */
+ target->hole.color = I915_COLOR_UNEVICTABLE;
+ target->hole.start = start;
+ target->hole.size = end;
+
+ list_for_each_entry(vma, &eviction_list, vm_link) {
+ target->hole.start =
+ min(target->hole.start, vma->node.start);
+ target->hole.size =
+ max(target->hole.size, node_end(&vma->node));
+
+ GEM_BUG_ON(vma->node.mm != &vm->mm);
+ drm_mm_remove_node(&vma->node);
+ atomic_and(~I915_VMA_BIND_MASK, &vma->flags);
+ GEM_BUG_ON(i915_vma_is_pinned(vma));
+ }
+ list_splice(&eviction_list, &work->eviction_list);
+
+ target->hole.size -= target->hole.start;
+
+ return drm_mm_reserve_node(&vm->mm, &target->hole) == 0;
+
+err:
+ list_splice(&eviction_list, &vm->bound_list);
+ return false;
+}
+
+static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
+{
+ u64 range, addr;
+
+ GEM_BUG_ON(range_overflows(start, len, end));
+ GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
+
+ range = round_down(end - len, align) - round_up(start, align);
+ if (range) {
+ if (sizeof(unsigned long) == sizeof(u64)) {
+ addr = get_random_long();
+ } else {
+ addr = get_random_int();
+ if (range > U32_MAX) {
+ addr <<= 32;
+ addr |= get_random_int();
+ }
+ }
+ div64_u64_rem(addr, range, &addr);
+ start += addr;
+ }
+
+ return round_up(start, align);
+}
+
+static int eb_reserve_vma(struct eb_vm_work *work, struct eb_vma *ev)
{
struct drm_i915_gem_exec_object2 *entry = ev->exec;
unsigned int exec_flags = ev->flags;
struct i915_vma *vma = ev->vma;
+ struct i915_address_space *vm = vma->vm;
+ u64 start = 0, end = vm->total;
+ u64 align = entry->alignment ?: I915_GTT_MIN_ALIGNMENT;
+ unsigned int bind_flags;
int err;
+ bind_flags = PIN_USER;
if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
- pin_flags |= PIN_GLOBAL;
+ bind_flags |= PIN_GLOBAL;
+
+ if (drm_mm_node_allocated(&vma->node))
+ goto pin;
+
+ vma->node.start = entry->offset & PIN_OFFSET_MASK;
+ vma->node.size = max(entry->pad_to_size, vma->size);
+ vma->node.color = 0;
+ if (i915_vm_has_cache_coloring(vm))
+ vma->node.color = vma->obj->cache_level;
/*
* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
* limit address to the first 4GBs for unflagged objects.
*/
if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
- pin_flags |= PIN_ZONE_4G;
+ end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
+
+ align = max(align, vma->display_alignment);
+ if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) {
+ vma->node.size = max_t(u64, vma->node.size, vma->fence_size);
+ end = min_t(u64, end, i915_vm_to_ggtt(vm)->mappable_end);
+ align = max_t(u64, align, vma->fence_alignment);
+ }
+
+ if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
+ start = BATCH_OFFSET_BIAS;
+
+ if (vma->node.size > end - start)
+ return -E2BIG;
+
+ if (vma->node.start >= start &&
+ IS_ALIGNED(vma->node.start, align) &&
+ !range_overflows(vma->node.start, vma->node.size, end)) {
+ unsigned int pin_flags;
- if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
- pin_flags |= PIN_MAPPABLE;
+ if (drm_mm_reserve_node(&vm->mm, &vma->node) == 0)
+ goto pin;
+
+ pin_flags = 0;
+ if (!(exec_flags & EXEC_OBJECT_PINNED))
+ pin_flags = PIN_NONBLOCK;
+
+ if (evict_for_node(work, ev, pin_flags))
+ goto pin;
+ }
if (exec_flags & EXEC_OBJECT_PINNED)
- pin_flags |= entry->offset | PIN_OFFSET_FIXED;
- else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
- pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+ return -ENOSPC;
+
+ vma->node.start = random_offset(start, end, vma->node.size, align);
+ if (evict_for_node(work, ev, PIN_NONBLOCK))
+ goto pin;
+
+ if (drm_mm_insert_node_in_range(&vm->mm, &vma->node,
+ vma->node.size,
+ align <= I915_GTT_MIN_ALIGNMENT ? 0 : align,
+ vma->node.color,
+ start, end, 0) == 0)
+ goto pin;
+
+#if 0
+ if (i915_gem_evict_something(vm,
+ vma->node.size,
+ align,
+ vma->node.color,
+ start, end, 0) == 0)
+ goto pin;
+#endif
- if (drm_mm_node_allocated(&vma->node) &&
- eb_vma_misplaced(entry, vma, ev->flags)) {
- err = i915_vma_unbind(vma);
- if (err)
- return err;
+ vma->node.start = random_offset(start, end, vma->node.size, align);
+ if (evict_for_node(work, ev, 0))
+ goto pin;
+
+ return -ENOSPC;
+
+pin:
+ __i915_vma_pin(vma);
+
+ ev->bind_flags = bind_flags & ~atomic_read(&vma->flags);
+ if (ev->bind_flags) {
+ i915_active_set_exclusive(&vma->active, &work->base.dma);
+ atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
+ atomic_or(bind_flags, &vma->flags);
}
- err = i915_vma_pin(vma,
- entry->pad_to_size, entry->alignment,
- pin_flags);
- if (err)
- return err;
+ GEM_BUG_ON(!ev->bind_flags && !drm_mm_node_allocated(&vma->node));
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
+ drm_mm_node_allocated(&ev->hole));
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node) &&
+ !drm_mm_node_allocated(&ev->hole));
if (entry->offset != vma->node.start) {
entry->offset = vma->node.start | UPDATE;
- eb->args->flags |= __EXEC_HAS_RELOC;
+ *work->p_flags |= __EXEC_HAS_RELOC;
}
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma); /* XXX no waiting */
if (unlikely(err)) {
- i915_vma_unpin(vma);
+ __i915_vma_unpin(vma);
return err;
}
@@ -650,44 +922,236 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
return 0;
}
+static int eb_bind_vma(struct dma_fence_work *base)
+{
+ struct eb_vm_work *work = container_of(base, typeof(*work), base);
+ struct i915_address_space *vm = work->vm;
+ struct eb_vma *ev;
+ int err = 0;
+
+ if (!list_empty(&work->eviction_list)) {
+ struct i915_vma *vma, *vn;
+
+ mutex_lock(&vm->mutex);
+ list_for_each_entry_safe(vma, vn,
+ &work->eviction_list, vm_link) {
+ GEM_BUG_ON(vma->vm != vm);
+ GEM_BUG_ON(!i915_vma_is_active(vma));
+
+ __evict(vma);
+ INIT_LIST_HEAD(&vma->vm_link);
+ }
+ mutex_unlock(&vm->mutex);
+ }
+
+ list_for_each_entry(ev, &work->unbound, bind_link) {
+ struct i915_vma *vma = ev->vma;
+
+ GEM_BUG_ON(vma->vm != vm);
+
+ if (!ev->bind_flags)
+ goto put;
+
+ GEM_BUG_ON(!i915_vma_is_active(vma));
+
+ if (err == 0)
+ err = vma->ops->bind_vma(vma,
+ vma->obj->cache_level,
+ ev->bind_flags |
+ I915_VMA_ALLOC);
+
+ GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND));
+ if (drm_mm_node_allocated(&ev->hole)) {
+ mutex_lock(&vm->mutex);
+ GEM_BUG_ON(ev->hole.mm != &vm->mm);
+ GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+ drm_mm_remove_node(&ev->hole);
+ drm_mm_reserve_node(&vm->mm, &vma->node);
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ list_move_tail(&vma->vm_link, &vm->bound_list);
+ mutex_unlock(&vm->mutex);
+ }
+ if (err) {
+ atomic_and(~ev->bind_flags, &vma->flags);
+ set_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma));
+ }
+
+put:
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
+ i915_vma_put(vma);
+ }
+
+ return err;
+}
+
+static void eb_vma_work_release(struct dma_fence_work *base)
+{
+ struct eb_vm_work *work = container_of(base, typeof(*work), base);
+
+ if (work->active)
+ i915_active_release(&work->vm->active);
+
+ eb_vma_array_put(work->array);
+}
+
+static const struct dma_fence_work_ops eb_bind_ops = {
+ .name = "eb_bind",
+ .work = eb_bind_vma,
+ .release = eb_vma_work_release,
+};
+
+static struct eb_vm_work *eb_vm_work(struct i915_execbuffer *eb)
+{
+ struct eb_vm_work *work;
+
+ work = kzalloc(sizeof(*work), GFP_KERNEL);
+ if (!work)
+ return NULL;
+
+ dma_fence_work_init(&work->base, &eb_bind_ops);
+ list_replace_init(&eb->unbound, &work->unbound);
+ work->array = eb_vma_array_get(eb->array);
+ work->p_flags = &eb->args->flags;
+ work->vm = eb->context->vm;
+
+ /* Preallocate our slot in vm->active, outside of vm->mutex */
+ work->active = i915_gem_context_async_id(eb->gem_context);
+ if (i915_active_acquire_for_context(&work->vm->active, work->active)) {
+ work->active = 0;
+ dma_fence_work_commit(&work->base);
+ return NULL;
+ }
+
+ INIT_LIST_HEAD(&work->eviction_list);
+
+ GEM_BUG_ON(list_empty(&work->unbound));
+ GEM_BUG_ON(!list_empty(&eb->unbound));
+
+ return work;
+}
+
+static int eb_vm_throttle(struct eb_vm_work *work)
+{
+ struct dma_fence *p;
+ int err;
+
+ /* Keep async work queued per context */
+ p = __i915_active_ref(&work->vm->active, work->active, &work->base.dma);
+ if (IS_ERR_OR_NULL(p))
+ return PTR_ERR(p);
+
+ err = i915_sw_fence_await_dma_fence(&work->base.chain, p, 0,
+ GFP_NOWAIT | __GFP_NOWARN);
+ dma_fence_put(p);
+
+ return err;
+}
+
+static int eb_prepare_vma(struct eb_vma *ev)
+{
+ struct i915_vma *vma = ev->vma;
+ int err;
+
+ GEM_BUG_ON(drm_mm_node_allocated(&ev->hole));
+ ev->bind_flags = 0;
+
+ /* Wait for a previous unbind to avoid reusing an active vma->node */
+ err = i915_vma_wait_for_unbind(vma);
+ if (err)
+ return err;
+
+ err = i915_vma_get_pages(vma);
+ if (err)
+ return err;
+
+ ev->flags |= __EXEC_OBJECT_HAS_PAGES;
+ return 0;
+}
+
static int eb_reserve(struct i915_execbuffer *eb)
{
const unsigned int count = eb->buffer_count;
- unsigned int pin_flags = PIN_USER | PIN_NONBLOCK;
+ struct i915_address_space *vm = eb->context->vm;
struct list_head last;
- struct eb_vma *ev;
unsigned int i, pass;
+ struct eb_vma *ev;
int err = 0;
- /*
- * Attempt to pin all of the buffers into the GTT.
- * This is done in 3 phases:
- *
- * 1a. Unbind all objects that do not match the GTT constraints for
- * the execbuffer (fenceable, mappable, alignment etc).
- * 1b. Increment pin count for already bound objects.
- * 2. Bind new objects.
- * 3. Decrement pin count.
- *
- * This avoid unnecessary unbinding of later objects in order to make
- * room for the earlier objects *unless* we need to defragment.
- */
-
- if (mutex_lock_interruptible(&eb->i915->drm.struct_mutex))
- return -EINTR;
-
pass = 0;
do {
+ struct eb_vm_work *work;
+
list_for_each_entry(ev, &eb->unbound, bind_link) {
- err = eb_reserve_vma(eb, ev, pin_flags);
+ err = eb_prepare_vma(ev);
+ switch (err) {
+ case 0:
+ break;
+ case -EAGAIN:
+ goto retry;
+ default:
+ return err;
+ }
+ }
+
+ work = eb_vm_work(eb);
+ if (!work)
+ return -ENOMEM;
+
+ if (mutex_lock_interruptible(&vm->mutex)) {
+ dma_fence_work_commit(&work->base);
+ return -EINTR;
+ }
+
+ err = eb_vm_throttle(work);
+ if (err) {
+ mutex_unlock(&vm->mutex);
+ dma_fence_work_commit(&work->base);
+ return err;
+ }
+
+ list_for_each_entry(ev, &work->unbound, bind_link) {
+ struct i915_vma *vma = ev->vma;
+
+ if (drm_mm_node_allocated(&vma->node) &&
+ eb_vma_misplaced(ev->exec, vma, ev->flags)) {
+ err = __i915_vma_unbind(vma);
+ if (err)
+ break;
+ }
+
+ if (!drm_mm_node_allocated(&vma->node) &&
+ i915_vma_is_active(vma)) {
+ err = -ENOSPC;
+ break;
+ }
+
+ err = i915_active_acquire(&vma->active);
+ if (!err) {
+ err = eb_reserve_vma(work, ev);
+ i915_active_release(&vma->active);
+ }
if (err)
break;
}
- if (!(err == -ENOSPC || err == -EAGAIN))
- break;
+
+ mutex_unlock(&vm->mutex);
+
+ dma_fence_get(&work->base.dma);
+ dma_fence_work_commit(&work->base);
+ if (err == -ENOSPC && dma_fence_wait(&work->base.dma, true))
+ err = -EINTR;
+ dma_fence_put(&work->base.dma);
+ if (err != -ENOSPC)
+ return err;
+
+retry:
+ if (!list_empty(&eb->unbound)) {
+ list_for_each_entry(ev, &eb->unbound, bind_link)
+ i915_vma_put(ev->vma);
+ INIT_LIST_HEAD(&eb->unbound);
+ }
/* Resort *all* the objects into priority order */
- INIT_LIST_HEAD(&eb->unbound);
INIT_LIST_HEAD(&last);
for (i = 0; i < count; i++) {
unsigned int flags;
@@ -711,13 +1175,13 @@ static int eb_reserve(struct i915_execbuffer *eb)
list_add(&ev->bind_link, &last);
else
list_add_tail(&ev->bind_link, &last);
+
+ i915_vma_get(ev->vma);
}
list_splice_tail(&last, &eb->unbound);
if (err == -EAGAIN) {
- mutex_unlock(&eb->i915->drm.struct_mutex);
flush_workqueue(eb->i915->mm.userptr_wq);
- mutex_lock(&eb->i915->drm.struct_mutex);
continue;
}
@@ -727,24 +1191,14 @@ static int eb_reserve(struct i915_execbuffer *eb)
case 1:
/* Too fragmented, unbind everything and retry */
- mutex_lock(&eb->context->vm->mutex);
- err = i915_gem_evict_vm(eb->context->vm);
- mutex_unlock(&eb->context->vm->mutex);
- if (err)
- goto unlock;
+ if (i915_active_wait(&vm->active))
+ return -EINTR;
break;
default:
- err = -ENOSPC;
- goto unlock;
+ return err;
}
-
- pin_flags = PIN_USER;
} while (1);
-
-unlock:
- mutex_unlock(&eb->i915->drm.struct_mutex);
- return err;
}
static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
@@ -784,10 +1238,6 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
if (unlikely(i915_gem_context_is_closed(eb->gem_context)))
return -ENOENT;
- INIT_LIST_HEAD(&eb->relocs);
- INIT_LIST_HEAD(&eb->unbound);
- INIT_LIST_HEAD(&eb->lock);
-
batch = eb_batch_index(eb);
for (i = 0; i < eb->buffer_count; i++) {
@@ -892,8 +1342,13 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
static void eb_destroy(const struct i915_execbuffer *eb)
{
+ struct eb_vma *ev;
+
GEM_BUG_ON(eb->reloc_cache.rq);
+ list_for_each_entry(ev, &eb->unbound, bind_link)
+ i915_vma_put(ev->vma);
+
eb_vma_array_put(eb->array);
if (eb->lut_size > 0)
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c
index 2a72cce63fd9..82d4f943c346 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.c
@@ -194,6 +194,8 @@ void __i915_vm_close(struct i915_address_space *vm)
void i915_address_space_fini(struct i915_address_space *vm)
{
+ i915_active_fini(&vm->active);
+
spin_lock(&vm->free_pages.lock);
if (pagevec_count(&vm->free_pages.pvec))
vm_free_pages_release(vm, true);
@@ -246,6 +248,8 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass)
drm_mm_init(&vm->mm, 0, vm->total);
vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
+ i915_active_init(&vm->active, NULL, NULL);
+
stash_init(&vm->free_pages);
INIT_LIST_HEAD(&vm->bound_list);
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h
index b3116fe8d180..7d4bfd764aee 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.h
@@ -262,6 +262,8 @@ struct i915_address_space {
*/
struct list_head bound_list;
+ struct i915_active active;
+
struct pagestash free_pages;
/* Global GTT */
diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
index f86f7e68ce5e..ecdd58f4b993 100644
--- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
@@ -162,7 +162,8 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
u32 pte_flags;
int err;
- if (flags & I915_VMA_ALLOC) {
+ if (flags & I915_VMA_ALLOC &&
+ !test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) {
err = vma->vm->allocate_va_range(vma->vm,
vma->node.start, vma->size);
if (err)
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index ed65d92a7609..f24385ee77e8 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -132,6 +132,7 @@ vma_create(struct drm_i915_gem_object *obj,
fs_reclaim_release(GFP_KERNEL);
}
+ INIT_LIST_HEAD(&vma->vm_link);
INIT_LIST_HEAD(&vma->closed_link);
if (view && view->type != I915_GGTT_VIEW_NORMAL) {
@@ -340,25 +341,32 @@ struct i915_vma_work *i915_vma_work(void)
return vw;
}
-int i915_vma_wait_for_bind(struct i915_vma *vma)
+static int __i915_vma_wait_excl(struct i915_vma *vma, bool bound)
{
+ struct dma_fence *fence;
int err = 0;
- if (rcu_access_pointer(vma->active.excl.fence)) {
- struct dma_fence *fence;
+ fence = i915_active_fence_get(&vma->active.excl);
+ if (!fence)
+ return 0;
- rcu_read_lock();
- fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
- rcu_read_unlock();
- if (fence) {
- err = dma_fence_wait(fence, MAX_SCHEDULE_TIMEOUT);
- dma_fence_put(fence);
- }
- }
+ if (drm_mm_node_allocated(&vma->node) == bound)
+ err = dma_fence_wait(fence, true);
+ dma_fence_put(fence);
return err;
}
+int i915_vma_wait_for_bind(struct i915_vma *vma)
+{
+ return __i915_vma_wait_excl(vma, true);
+}
+
+int i915_vma_wait_for_unbind(struct i915_vma *vma)
+{
+ return __i915_vma_wait_excl(vma, false);
+}
+
/**
* i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
* @vma: VMA to map
@@ -726,7 +734,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
- list_add_tail(&vma->vm_link, &vma->vm->bound_list);
+ list_move_tail(&vma->vm_link, &vma->vm->bound_list);
return 0;
}
@@ -742,7 +750,7 @@ i915_vma_detach(struct i915_vma *vma)
* vma, we can drop its hold on the backing storage and allow
* it to be reaped by the shrinker.
*/
- list_del(&vma->vm_link);
+ list_del_init(&vma->vm_link);
}
static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
@@ -788,7 +796,7 @@ static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
return pinned;
}
-static int vma_get_pages(struct i915_vma *vma)
+int i915_vma_get_pages(struct i915_vma *vma)
{
int err = 0;
@@ -835,7 +843,7 @@ static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
mutex_unlock(&vma->pages_mutex);
}
-static void vma_put_pages(struct i915_vma *vma)
+void i915_vma_put_pages(struct i915_vma *vma)
{
if (atomic_add_unless(&vma->pages_count, -1, 1))
return;
@@ -843,7 +851,7 @@ static void vma_put_pages(struct i915_vma *vma)
__vma_put_pages(vma, 1);
}
-static void vma_unbind_pages(struct i915_vma *vma)
+void i915_vma_unbind_pages(struct i915_vma *vma)
{
unsigned int count;
@@ -874,10 +882,14 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK))
return 0;
- err = vma_get_pages(vma);
+ err = i915_vma_get_pages(vma);
if (err)
return err;
+ err = i915_vma_wait_for_unbind(vma);
+ if (err)
+ goto err_pages;
+
if (flags & vma->vm->bind_async_flags) {
work = i915_vma_work();
if (!work) {
@@ -960,7 +972,7 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (wakeref)
intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
err_pages:
- vma_put_pages(vma);
+ i915_vma_put_pages(vma);
return err;
}
@@ -1108,17 +1120,6 @@ void i915_vma_parked(struct intel_gt *gt)
spin_unlock_irq(>->closed_lock);
}
-static void __i915_vma_iounmap(struct i915_vma *vma)
-{
- GEM_BUG_ON(i915_vma_is_pinned(vma));
-
- if (vma->iomap == NULL)
- return;
-
- io_mapping_unmap(vma->iomap);
- vma->iomap = NULL;
-}
-
void i915_vma_revoke_mmap(struct i915_vma *vma)
{
struct drm_vma_offset_node *node;
@@ -1273,7 +1274,7 @@ int __i915_vma_unbind(struct i915_vma *vma)
&vma->flags);
i915_vma_detach(vma);
- vma_unbind_pages(vma);
+ i915_vma_unbind_pages(vma);
drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
return 0;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index e1ced1df13e1..02d1fb98aa41 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -239,6 +239,10 @@ int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags);
+int i915_vma_get_pages(struct i915_vma *vma);
+void i915_vma_put_pages(struct i915_vma *vma);
+void i915_vma_unbind_pages(struct i915_vma *vma);
+
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;
@@ -293,6 +297,17 @@ static inline bool i915_node_color_differs(const struct drm_mm_node *node,
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
+static inline void __i915_vma_iounmap(struct i915_vma *vma)
+{
+ GEM_BUG_ON(i915_vma_is_pinned(vma));
+
+ if (!vma->iomap)
+ return;
+
+ io_mapping_unmap(vma->iomap);
+ vma->iomap = NULL;
+}
+
/**
* i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
* @vma: VMA to unpin
@@ -326,7 +341,7 @@ static inline struct page *i915_vma_first_page(struct i915_vma *vma)
* True if the vma has a fence, false otherwise.
*/
int __must_check i915_vma_pin_fence(struct i915_vma *vma);
-int __must_check i915_vma_revoke_fence(struct i915_vma *vma);
+int i915_vma_revoke_fence(struct i915_vma *vma);
int __i915_vma_pin_fence(struct i915_vma *vma);
@@ -376,6 +391,7 @@ void i915_vma_make_shrinkable(struct i915_vma *vma);
void i915_vma_make_purgeable(struct i915_vma *vma);
int i915_vma_wait_for_bind(struct i915_vma *vma);
+int i915_vma_wait_for_unbind(struct i915_vma *vma);
static inline int i915_vma_sync(struct i915_vma *vma)
{
--
2.20.1
More information about the Intel-gfx-trybot
mailing list