[PATCH 49/56] drm/i915: Move vma pinning under vm->mutex
Chris Wilson
chris at chris-wilson.co.uk
Sun Jul 1 17:44:48 UTC 2018
Use the local GTT lock (vm->mutex) for pinning the VMA (including
insertion and removal), as opposed to relying on the BKL struct_mutex.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_drv.h | 5 +
drivers/gpu/drm/i915/i915_gem.c | 137 ++++++----------
drivers/gpu/drm/i915/i915_gem_evict.c | 14 +-
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 52 ++++---
drivers/gpu/drm/i915/i915_gem_gtt.c | 147 ++++++++++++------
drivers/gpu/drm/i915/i915_gem_render_state.c | 17 +-
drivers/gpu/drm/i915/i915_gem_shrinker.c | 2 +
drivers/gpu/drm/i915/i915_gem_stolen.c | 8 +-
drivers/gpu/drm/i915/i915_gem_tiling.c | 30 ++--
drivers/gpu/drm/i915/i915_vma.c | 109 ++++++-------
drivers/gpu/drm/i915/i915_vma.h | 39 +++--
drivers/gpu/drm/i915/intel_engine_cs.c | 13 +-
drivers/gpu/drm/i915/intel_ringbuffer.c | 3 +-
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 2 +-
14 files changed, 329 insertions(+), 249 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 57a99545e770..ca6260b546b4 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -953,6 +953,11 @@ struct i915_gem_mm {
*/
struct pagevec wc_stash;
+ /**
+ * Lock for the small stash of WC pages.
+ */
+ spinlock_t wc_lock;
+
/**
* tmpfs instance used for shmem backed objects
*/
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ad8136a859e4..001c0fcbee49 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -67,17 +67,26 @@ static int
insert_mappable_node(struct i915_ggtt *ggtt,
struct drm_mm_node *node, u32 size)
{
+ int err;
+
memset(node, 0, sizeof(*node));
- return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
- size, 0, I915_COLOR_UNEVICTABLE,
- 0, ggtt->mappable_end,
- DRM_MM_INSERT_LOW);
+
+ mutex_lock(&ggtt->vm.mutex);
+ err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
+ size, 0, I915_COLOR_UNEVICTABLE,
+ 0, ggtt->mappable_end,
+ DRM_MM_INSERT_LOW);
+ mutex_unlock(&ggtt->vm.mutex);
+
+ return err;
}
static void
-remove_mappable_node(struct drm_mm_node *node)
+remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
{
+ mutex_lock(&ggtt->vm.mutex);
drm_mm_remove_node(node);
+ mutex_unlock(&ggtt->vm.mutex);
}
/* some bookkeeping */
@@ -445,7 +454,9 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
list_move_tail(&vma->obj_link, &still_in_list);
spin_unlock(&obj->vma.lock);
+ mutex_lock(&vma->vm->mutex);
ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
spin_lock(&obj->vma.lock);
}
@@ -1247,7 +1258,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
@@ -1455,7 +1466,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
@@ -2101,17 +2112,19 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
goto err_fence;
/* Mark as being mmapped into userspace for later revocation */
+ mutex_lock(&vma->vm->mutex);
assert_rpm_wakelock_held(dev_priv);
if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
GEM_BUG_ON(!obj->userfault_count);
i915_vma_set_ggtt_write(vma);
+ mutex_unlock(&vma->vm->mutex);
err_fence:
i915_vma_unpin_fence(vma);
err_unpin:
- __i915_vma_unpin(vma);
+ i915_vma_unpin(vma);
err_unlock:
mutex_unlock(&dev->struct_mutex);
err_rpm:
@@ -3590,11 +3603,13 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (obj->cache_level == cache_level)
return 0;
- /* Inspect the list of currently bound VMA and unbind any that would
+ /*
+ * Inspect the list of currently bound VMA and unbind any that would
* be invalid given the new cache-level. This is principally to
* catch the issue of the CS prefetch crossing page boundaries and
* reading an invalid PTE on older architectures.
*/
+ spin_lock(&obj->vma.lock);
restart:
list_for_each_entry(vma, &obj->vma.list, obj_link) {
if (!drm_mm_node_allocated(&vma->node))
@@ -3602,16 +3617,17 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (i915_vma_is_pinned(vma)) {
DRM_DEBUG("can not change the cache level of pinned objects\n");
- return -EBUSY;
+ ret = -EBUSY;
+ break;
}
- if (!i915_vma_is_closed(vma) &&
- i915_gem_valid_gtt_space(vma, cache_level))
- continue;
-
+ spin_unlock(&obj->vma.lock);
+ mutex_lock(&vma->vm->mutex);
ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ spin_lock(&obj->vma.lock);
if (ret)
- return ret;
+ break;
/* As unbinding may affect other elements in the
* obj->vma_list (due to side-effects from retiring
@@ -3619,70 +3635,9 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
*/
goto restart;
}
-
- /* We can reuse the existing drm_mm nodes but need to change the
- * cache-level on the PTE. We could simply unbind them all and
- * rebind with the correct cache-level on next use. However since
- * we already have a valid slot, dma mapping, pages etc, we may as
- * rewrite the PTE in the belief that doing so tramples upon less
- * state and so involves less work.
- */
- if (obj->bind_count) {
- /* Before we change the PTE, the GPU must not be accessing it.
- * If we wait upon the object, we know that all the bound
- * VMA are no longer active.
- */
- ret = i915_gem_object_wait(obj,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
- I915_WAIT_ALL,
- MAX_SCHEDULE_TIMEOUT,
- NULL);
- if (ret)
- return ret;
-
- if (!HAS_LLC(to_i915(obj->base.dev)) &&
- cache_level != I915_CACHE_NONE) {
- /* Access to snoopable pages through the GTT is
- * incoherent and on some machines causes a hard
- * lockup. Relinquish the CPU mmaping to force
- * userspace to refault in the pages and we can
- * then double check if the GTT mapping is still
- * valid for that pointer access.
- */
- i915_gem_release_mmap(obj);
-
- /* As we no longer need a fence for GTT access,
- * we can relinquish it now (and so prevent having
- * to steal a fence from someone else on the next
- * fence request). Note GPU activity would have
- * dropped the fence as all snoopable access is
- * supposed to be linear.
- */
- for_each_ggtt_vma(vma, obj) {
- ret = i915_vma_put_fence(vma);
- if (ret)
- return ret;
- }
- } else {
- /* We either have incoherent backing store and
- * so no GTT access or the architecture is fully
- * coherent. In such cases, existing GTT mmaps
- * ignore the cache bit in the PTE and we can
- * rewrite it without confusing the GPU or having
- * to force userspace to fault back in its mmaps.
- */
- }
-
- list_for_each_entry(vma, &obj->vma.list, obj_link) {
- if (!drm_mm_node_allocated(&vma->node))
- continue;
-
- ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
- if (ret)
- return ret;
- }
- }
+ spin_unlock(&obj->vma.lock);
+ if (ret)
+ return ret;
list_for_each_entry(vma, &obj->vma.list, obj_link)
vma->node.color = cache_level;
@@ -3991,8 +3946,6 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
if (flags & PIN_MAPPABLE &&
(!view || view->type == I915_GGTT_VIEW_NORMAL)) {
/* If the required space is larger than the available
@@ -4029,14 +3982,18 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
if (unlikely(IS_ERR(vma)))
return vma;
+ mutex_lock(&vm->mutex);
+
if (i915_vma_misplaced(vma, size, alignment, flags)) {
if (flags & PIN_NONBLOCK) {
+ ret = -ENOSPC;
+
if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
- return ERR_PTR(-ENOSPC);
+ goto err_unlock;
if (flags & PIN_MAPPABLE &&
vma->fence_size > dev_priv->ggtt.mappable_end / 2)
- return ERR_PTR(-ENOSPC);
+ goto err_unlock;
}
WARN(i915_vma_is_pinned(vma),
@@ -4046,16 +4003,22 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
i915_ggtt_offset(vma), alignment,
!!(flags & PIN_MAPPABLE),
i915_vma_is_map_and_fenceable(vma));
+
ret = i915_vma_unbind(vma);
if (ret)
- return ERR_PTR(ret);
+ goto err_unlock;
}
- ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
+ ret = __i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
if (ret)
- return ERR_PTR(ret);
+ goto err_unlock;
+ mutex_unlock(&vm->mutex);
return vma;
+
+err_unlock:
+ mutex_unlock(&vm->mutex);
+ return ERR_PTR(ret);
}
static __always_inline unsigned int __busy_read_flag(unsigned int id)
@@ -4974,7 +4937,9 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
* from the GTT to prevent such accidents and reclaim the
* space.
*/
+ mutex_lock(&state->vm->mutex);
err = i915_vma_unbind(state);
+ mutex_unlock(&state->vm->mutex);
if (err)
goto err_active;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 23ee248a1a65..fdb83336e95a 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -63,7 +63,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
* the hopes that we can then remove contexts and the like only
* bound by their active reference.
*/
- err = i915_gem_switch_to_kernel_context(i915, GFP_KERNEL);
+ err = i915_gem_switch_to_kernel_context(i915, GFP_NOWAIT);
if (err)
return err;
@@ -132,7 +132,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
struct i915_vma *active;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
@@ -247,7 +247,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
*/
list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
if (drm_mm_scan_remove_block(&scan, &vma->node))
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
else
list_del(&vma->evict_link);
}
@@ -291,7 +291,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
bool check_color;
int ret = 0;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
@@ -371,7 +371,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
* unbinding one vma from freeing (by dropping its active
* reference) another in our eviction list.
*/
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
@@ -402,7 +402,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
struct i915_vma *vma, *next;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict_vm(vm);
/* Switch back to the default context in order to unpin
@@ -421,7 +421,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
if (i915_vma_is_pinned(vma))
continue;
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 2021de82ba50..32cbdfd4f16d 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -405,7 +405,7 @@ eb_pin_vma(struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
pin_flags |= PIN_GLOBAL;
- if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
+ if (unlikely(__i915_vma_pin(vma, 0, 0, pin_flags)))
return false;
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
@@ -617,9 +617,9 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
}
- err = i915_vma_pin(vma,
- entry->pad_to_size, entry->alignment,
- pin_flags);
+ err = __i915_vma_pin(vma,
+ entry->pad_to_size, entry->alignment,
+ pin_flags);
if (err)
return err;
@@ -631,7 +631,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
err = i915_vma_reserve_fence(vma);
if (unlikely(err)) {
- i915_vma_unpin(vma);
+ __i915_vma_unpin(vma);
return err;
}
@@ -762,6 +762,7 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
batch = eb_batch_index(eb);
+ mutex_lock(&eb->vm->mutex);
for (i = 0; i < eb->buffer_count; i++) {
u32 handle = eb->exec[i].handle;
struct i915_lut_handle *lut;
@@ -771,6 +772,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
if (likely(vma))
goto add_vma;
+ mutex_unlock(&eb->vm->mutex);
+
obj = i915_gem_object_lookup(eb->file, handle);
if (unlikely(!obj)) {
err = -ENOENT;
@@ -795,6 +798,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
goto err_obj;
}
+ mutex_lock(&eb->vm->mutex);
+
/* transfer ref to ctx */
if (!vma->open_count++)
i915_vma_reopen(vma);
@@ -805,8 +810,10 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
add_vma:
err = eb_add_vma(eb, i, batch, vma);
- if (unlikely(err))
+ if (unlikely(err)) {
+ mutex_unlock(&eb->vm->mutex);
goto err_vma;
+ }
GEM_BUG_ON(vma != eb->vma[i]);
GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
@@ -815,7 +822,9 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
}
eb->args->flags |= __EXEC_VALIDATED;
- return eb_reserve(eb);
+ err = eb_reserve(eb);
+ mutex_unlock(&eb->vm->mutex);
+ return err;
err_obj:
i915_gem_object_put(obj);
@@ -849,6 +858,7 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
const unsigned int count = eb->buffer_count;
unsigned int i;
+ mutex_lock(&eb->vm->mutex);
for (i = 0; i < count; i++) {
struct i915_vma *vma = eb->vma[i];
unsigned int flags = eb->flags[i];
@@ -860,12 +870,20 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
vma->exec_flags = NULL;
eb->vma[i] = NULL;
+ if (unlikely(vma->vm != eb->vm))
+ mutex_lock_nested(&vma->vm->mutex,
+ SINGLE_DEPTH_NESTING);
+
if (flags & __EXEC_OBJECT_HAS_PIN)
__eb_unreserve_vma(vma, flags);
if (flags & __EXEC_OBJECT_HAS_REF)
i915_vma_put(vma);
+
+ if (unlikely(vma->vm != eb->vm))
+ mutex_unlock(&vma->vm->mutex);
}
+ mutex_unlock(&eb->vm->mutex);
}
static void eb_reset_vmas(const struct i915_execbuffer *eb)
@@ -1364,8 +1382,14 @@ eb_relocate_entry(struct i915_execbuffer *eb,
*/
if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
IS_GEN6(eb->i915)) {
+ reloc_cache_reset(&eb->reloc_cache);
+
+ GEM_BUG_ON(!i915_vma_is_ggtt(target));
+ mutex_lock(&target->vm->mutex);
err = i915_vma_bind(target, target->obj->cache_level,
PIN_GLOBAL);
+ mutex_unlock(&target->vm->mutex);
+
if (WARN_ONCE(err,
"Unexpected failure to bind target VMA!"))
return err;
@@ -1829,22 +1853,14 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
}
for (i = 0; i < count; i++) {
- unsigned int flags = eb->flags[i];
- struct i915_vma *vma = eb->vma[i];
-
- err = i915_vma_move_to_active(vma, eb->request, flags);
+ err = i915_vma_move_to_active(eb->vma[i],
+ eb->request,
+ eb->flags[i]);
if (unlikely(err)) {
i915_request_skip(eb->request, err);
return err;
}
-
- __eb_unreserve_vma(vma, flags);
- vma->exec_flags = NULL;
-
- if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
- i915_vma_put(vma);
}
- eb->exec = NULL;
/* Unconditionally flush any chipset caches (for streaming writes). */
i915_gem_chipset_flush(eb->i915);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 081bc18e4507..2307a04e9f0c 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -383,6 +383,7 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
{
struct pagevec *pvec = &vm->free_pages;
struct pagevec stash;
+ struct page *page;
if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
i915_gem_shrink_all(vm->i915);
@@ -393,13 +394,15 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
if (!vm->pt_kmap_wc)
return alloc_page(gfp);
- /* A placeholder for a specific mutex to guard the WC stash */
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
-
/* Look in our global stash of WC pages... */
pvec = &vm->i915->mm.wc_stash;
+ page = NULL;
+ spin_lock(&vm->i915->mm.wc_lock);
if (likely(pvec->nr))
- return pvec->pages[--pvec->nr];
+ page = pvec->pages[--pvec->nr];
+ spin_unlock(&vm->i915->mm.wc_lock);
+ if (page)
+ return page;
/*
* Otherwise batch allocate pages to amoritize cost of set_pages_wc.
@@ -434,11 +437,14 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
pagevec_release(&stash);
}
- return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
+ page = NULL;
+ if (likely(pvec->nr))
+ page = pvec->pages[--pvec->nr];
+ return page;
}
static void vm_free_pages_release(struct i915_address_space *vm,
- bool immediate)
+ unsigned int immediate)
{
struct pagevec *pvec = &vm->free_pages;
@@ -446,28 +452,32 @@ static void vm_free_pages_release(struct i915_address_space *vm,
if (vm->pt_kmap_wc) {
struct pagevec *stash = &vm->i915->mm.wc_stash;
+ spinlock_t *lock = &vm->i915->mm.wc_lock;
- /* When we use WC, first fill up the global stash and then
+ /*
+ * When we use WC, first fill up the global stash and then
* only if full immediately free the overflow.
*/
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ spin_lock(lock);
if (pagevec_space(stash)) {
do {
stash->pages[stash->nr++] =
pvec->pages[--pvec->nr];
if (!pvec->nr)
- return;
+ break;
} while (pagevec_space(stash));
-
- /* As we have made some room in the VM's free_pages,
- * we can wait for it to fill again. Unless we are
- * inside i915_address_space_fini() and must
- * immediately release the pages!
- */
- if (!immediate)
- return;
}
+ spin_unlock(lock);
+
+ /*
+ * As we have made some room in the VM's free_pages,
+ * we can wait for it to fill again. Unless we are
+ * inside i915_address_space_fini() and must
+ * immediately release the pages!
+ */
+ if (pvec->nr <= immediate)
+ return;
set_pages_array_wb(pvec->pages, pvec->nr);
}
@@ -486,7 +496,7 @@ static void vm_free_page(struct i915_address_space *vm, struct page *page)
*/
might_sleep();
if (!pagevec_add(&vm->free_pages, page))
- vm_free_pages_release(vm, false);
+ vm_free_pages_release(vm, PAGEVEC_SIZE - 1);
}
static int __setup_page_dma(struct i915_address_space *vm,
@@ -1332,17 +1342,24 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
if (pt == vm->scratch_pt) {
pd->used_pdes++;
+ mutex_unlock(&vm->mutex);
pt = alloc_pt(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pt)) {
pd->used_pdes--;
goto unwind;
}
- if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
- gen8_initialize_pt(vm, pt);
+ if (pd->page_table[pde] == vm->scratch_pt) {
+ if (count < GEN8_PTES ||
+ intel_vgpu_active(vm->i915))
+ gen8_initialize_pt(vm, pt);
- gen8_ppgtt_set_pde(vm, pd, pt, pde);
- GEM_BUG_ON(pd->used_pdes > I915_PDES);
+ gen8_ppgtt_set_pde(vm, pd, pt, pde);
+ GEM_BUG_ON(pd->used_pdes > I915_PDES);
+ } else {
+ free_pt(vm, pt);
+ }
}
pt->used_ptes += count;
@@ -1367,17 +1384,23 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
if (pd == vm->scratch_pd) {
pdp->used_pdpes++;
+ mutex_unlock(&vm->mutex);
pd = alloc_pd(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pd)) {
pdp->used_pdpes--;
goto unwind;
}
- gen8_initialize_pd(vm, pd);
- gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
- GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
+ if (pdp->page_directory[pdpe] == vm->scratch_pd) {
+ gen8_initialize_pd(vm, pd);
+ gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
+ GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
- mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
+ mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
+ } else {
+ free_pd(vm, pd);
+ }
}
ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
@@ -1418,12 +1441,18 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+ mutex_unlock(&vm->mutex);
pdp = alloc_pdp(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pdp))
goto unwind;
- gen8_initialize_pdp(vm, pdp);
- gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+ if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+ gen8_initialize_pdp(vm, pdp);
+ gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+ } else {
+ free_pdp(vm, pdp);
+ }
}
ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
@@ -2136,7 +2165,7 @@ static void i915_address_space_init(struct i915_address_space *vm,
static void i915_address_space_fini(struct i915_address_space *vm)
{
if (pagevec_count(&vm->free_pages))
- vm_free_pages_release(vm, true);
+ vm_free_pages_release(vm, 0);
drm_mm_takedown(&vm->mm);
list_del(&vm->global_link);
@@ -2963,8 +2992,10 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
mutex_lock(&dev_priv->drm.struct_mutex);
i915_gem_fini_aliasing_ppgtt(dev_priv);
+ mutex_lock(&ggtt->vm.mutex);
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&ggtt->vm.mutex);
if (drm_mm_node_allocated(&ggtt->error_capture))
drm_mm_remove_node(&ggtt->error_capture);
@@ -3578,6 +3609,9 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
struct i915_ggtt *ggtt = &dev_priv->ggtt;
int ret;
+ spin_lock_init(&dev_priv->mm.wc_lock);
+ pagevec_init(&dev_priv->mm.wc_stash);
+
INIT_LIST_HEAD(&dev_priv->vm_list);
/* Note that we use page colouring to enforce a guard page at the
@@ -3671,8 +3705,6 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
/* clflush objects bound into the GGTT and rebind them. */
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
- struct drm_i915_gem_object *obj = vma->obj;
-
if (!(vma->flags & I915_VMA_GLOBAL_BIND))
continue;
@@ -3680,10 +3712,8 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
continue;
WARN_ON(i915_vma_bind(vma,
- obj ? obj->cache_level : 0,
+ vma->obj ? vma->obj->cache_level : 0,
PIN_UPDATE));
- if (obj)
- WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
}
ggtt->vm.closed = false;
@@ -3691,6 +3721,13 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
mutex_unlock(&ggtt->vm.mutex);
+ list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
+ if (!vma->obj || !(vma->flags & I915_VMA_GLOBAL_BIND))
+ continue;
+
+ WARN_ON(i915_gem_object_set_to_gtt_domain(vma->obj, false));
+ }
+
if (INTEL_GEN(dev_priv) >= 8) {
struct intel_ppat *ppat = &dev_priv->ppat;
@@ -3842,9 +3879,10 @@ intel_partial_pages(const struct i915_ggtt_view *view,
static int
i915_get_ggtt_vma_pages(struct i915_vma *vma)
{
- int ret;
+ struct sg_table *pages;
- /* The vma->pages are only valid within the lifespan of the borrowed
+ /*
+ * The vma->pages are only valid within the lifespan of the borrowed
* obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
* must be the vma->pages. A simple rule is that vma->pages must only
* be accessed when the obj->mm.pages are pinned.
@@ -3856,27 +3894,35 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
GEM_BUG_ON(vma->ggtt_view.type);
/* fall through */
case I915_GGTT_VIEW_NORMAL:
- vma->pages = vma->obj->mm.pages;
- return 0;
+ pages = vma->obj->mm.pages;
+ goto out;
case I915_GGTT_VIEW_ROTATED:
- vma->pages =
- intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
+ case I915_GGTT_VIEW_PARTIAL:
+ break;
+ }
+
+ GEM_BUG_ON(!i915_vma_is_pinned(vma));
+ mutex_unlock(&vma->vm->mutex);
+ switch ((int)vma->ggtt_view.type) {
+ case I915_GGTT_VIEW_ROTATED:
+ pages = intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
break;
case I915_GGTT_VIEW_PARTIAL:
- vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
+ pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
+ break;
+ default:
+ pages = ERR_PTR(-ENODEV);
break;
}
+ mutex_lock(&vma->vm->mutex);
+ if (IS_ERR(pages))
+ return PTR_ERR(pages);
- ret = 0;
- if (unlikely(IS_ERR(vma->pages))) {
- ret = PTR_ERR(vma->pages);
- vma->pages = NULL;
- DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
- vma->ggtt_view.type, ret);
- }
- return ret;
+out:
+ vma->pages = pages;
+ return 0;
}
/**
@@ -3911,6 +3957,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm,
{
int err;
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
@@ -4004,7 +4051,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
u64 offset;
int err;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(alignment && !is_power_of_2(alignment));
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 90baf9086d0a..904581fdbde7 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -179,6 +179,7 @@ static int render_state_setup(struct intel_render_state *so,
int i915_gem_render_state_emit(struct i915_request *rq)
{
+ struct i915_ggtt *ggtt = &rq->i915->ggtt;
struct intel_engine_cs *engine = rq->engine;
struct intel_render_state so = {}; /* keep the compiler happy */
int err;
@@ -194,15 +195,19 @@ int i915_gem_render_state_emit(struct i915_request *rq)
if (IS_ERR(so.obj))
return PTR_ERR(so.obj);
- so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.vm, NULL);
+ so.vma = i915_vma_instance(so.obj, &ggtt->vm, NULL);
if (IS_ERR(so.vma)) {
err = PTR_ERR(so.vma);
goto err_obj;
}
- err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+ mutex_lock(&ggtt->vm.mutex);
+
+ err = __i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
if (err)
- goto err_vma;
+ goto err_vma_locked;
+
+ mutex_unlock(&ggtt->vm.mutex);
err = render_state_setup(&so, rq->i915);
if (err)
@@ -224,9 +229,11 @@ int i915_gem_render_state_emit(struct i915_request *rq)
err = i915_vma_move_to_active(so.vma, rq, 0);
err_unpin:
- i915_vma_unpin(so.vma);
-err_vma:
+ mutex_lock(&ggtt->vm.mutex);
+ __i915_vma_unpin(so.vma);
+err_vma_locked:
i915_vma_close(so.vma);
+ mutex_unlock(&ggtt->vm.mutex);
err_obj:
__i915_gem_object_release_unless_active(so.obj);
return err;
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 52c061b728ca..006a72b0d6a6 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -480,6 +480,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
intel_runtime_pm_put(i915);
/* We also want to clear any cached iomaps as they wrap vmap */
+ mutex_lock(&i915->ggtt.vm.mutex);
list_for_each_entry_safe(vma, next,
&i915->ggtt.vm.bound_list, vm_link) {
unsigned long count = vma->node.size >> PAGE_SHIFT;
@@ -490,6 +491,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
if (i915_vma_unbind(vma) == 0)
freed_pages += count;
}
+ mutex_unlock(&i915->ggtt.vm.mutex);
out:
shrinker_unlock(i915, unlock);
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index f65ee28d5d81..917bd92d76d7 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -600,8 +600,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return NULL;
- lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
&stolen_offset, >t_offset, &size);
@@ -648,7 +646,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
goto err_pages;
}
- /* To simplify the initialisation sequence between KMS and GTT,
+ mutex_lock(&vma->vm->mutex);
+
+ /*
+ * To simplify the initialisation sequence between KMS and GTT,
* we allow construction of the stolen object prior to
* setting up the GTT space. The actual reservation will occur
* later.
@@ -667,7 +668,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
vma->flags |= I915_VMA_GLOBAL_BIND;
__i915_vma_set_map_and_fenceable(vma);
- mutex_lock(&ggtt->vm.mutex);
list_add_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index d9dc9df523b5..8292492b1b86 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -199,22 +199,25 @@ static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
int tiling_mode, unsigned int stride)
{
+ struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
struct i915_vma *vma;
int ret;
if (tiling_mode == I915_TILING_NONE)
return 0;
+ mutex_lock(&ggtt->vm.mutex);
for_each_ggtt_vma(vma, obj) {
if (i915_vma_fence_prepare(vma, tiling_mode, stride))
continue;
ret = i915_vma_unbind(vma);
if (ret)
- return ret;
+ break;
}
+ mutex_unlock(&ggtt->vm.mutex);
- return 0;
+ return ret;
}
int
@@ -251,16 +254,20 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
* whilst executing a fenced command for an untiled object.
*/
- err = i915_gem_object_fence_prepare(obj, tiling, stride);
- if (err)
- return err;
-
i915_gem_object_lock(obj);
+
if (i915_gem_object_is_framebuffer(obj)) {
- i915_gem_object_unlock(obj);
- return -EBUSY;
+ err = -EBUSY;
+ goto unlock;
}
+ err = i915_gem_object_fence_prepare(obj, tiling, stride);
+ if (err)
+ goto unlock;
+
+ /* Force the fence to be reacquired for GTT access */
+ i915_gem_release_mmap(obj);
+
/* If the memory has unknown (i.e. varying) swizzling, we pin the
* pages to prevent them being swapped out and causing corruption
* due to the change in swizzling.
@@ -296,9 +303,6 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
obj->tiling_and_stride = tiling | stride;
i915_gem_object_unlock(obj);
- /* Force the fence to be reacquired for GTT access */
- i915_gem_release_mmap(obj);
-
/* Try to preallocate memory required to save swizzling on put-pages */
if (i915_gem_object_needs_bit17_swizzle(obj)) {
if (!obj->bit_17) {
@@ -311,6 +315,10 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
}
return 0;
+
+unlock:
+ i915_gem_object_unlock(obj);
+ return err;
}
/**
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 4cbb6f69c7da..0bcc91a06733 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -274,7 +274,7 @@ vma_lookup(struct drm_i915_gem_object *obj,
* Once created, the VMA is kept until either the object is freed, or the
* address space is closed.
*
- * Must be called with struct_mutex held.
+ * Must be called with vm->mutex held.
*
* Returns the vma, or an error pointer.
*/
@@ -316,6 +316,7 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
u32 vma_flags;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->size > vma->node.size);
@@ -359,8 +360,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
/* Access through the GTT requires the device to be awake. */
assert_rpm_wakelock_held(vma->vm->i915);
+ mutex_lock(&vma->vm->mutex);
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
err = -ENODEV;
goto err;
@@ -382,18 +383,20 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
vma->iomap = ptr;
}
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma);
if (err)
goto err_unpin;
i915_vma_set_ggtt_write(vma);
+ mutex_unlock(&vma->vm->mutex);
return ptr;
err_unpin:
__i915_vma_unpin(vma);
err:
+ mutex_unlock(&vma->vm->mutex);
return IO_ERR_PTR(err);
}
@@ -558,6 +561,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
u64 start, end;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(i915_vma_is_closed(vma));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
@@ -584,7 +589,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
- /* If binding the object/GGTT view requires more space than the entire
+ /*
+ * If binding the object/GGTT view requires more space than the entire
* aperture has, reject it early before evicting everything in a vain
* attempt to find space.
*/
@@ -596,11 +602,28 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
}
if (vma->obj) {
- ret = i915_gem_object_pin_pages(vma->obj);
+ struct drm_i915_gem_object *obj = vma->obj;
+
+ mutex_unlock(&vma->vm->mutex);
+ ret = i915_gem_object_pin_pages(obj);
+ mutex_lock(&vma->vm->mutex);
if (ret)
return ret;
- cache_level = vma->obj->cache_level;
+ /*
+ * Check that someone else didn't complete the job on our
+ * behalf while we dropped the lock.
+ */
+ if (drm_mm_node_allocated(&vma->node)) {
+ i915_gem_object_unpin_pages(obj);
+
+ if (i915_vma_misplaced(vma, size, alignment, flags))
+ return -EAGAIN;
+
+ return 0;
+ }
+
+ cache_level = obj->cache_level;
} else {
cache_level = 0;
}
@@ -670,9 +693,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
- mutex_lock(&vma->vm->mutex);
+ lockdep_assert_held(&vma->vm->mutex);
list_add_tail(&vma->vm_link, &vma->vm->bound_list);
- mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
@@ -700,15 +722,15 @@ i915_vma_remove(struct i915_vma *vma)
{
struct drm_i915_private *i915 = vma->vm->i915;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
vma->ops->clear_pages(vma);
- mutex_lock(&vma->vm->mutex);
drm_mm_remove_node(&vma->node);
list_del(&vma->vm_link);
- mutex_unlock(&vma->vm->mutex);
/*
* Since the unbound list is global, only move to that list if
@@ -738,7 +760,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
const unsigned int bound = vma->flags;
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
@@ -841,6 +863,8 @@ static void __i915_vma_destroy(struct i915_vma *vma)
void i915_vma_destroy(struct i915_vma *vma)
{
+ struct i915_address_space *vm = vma->vm;
+
lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
GEM_BUG_ON(i915_vma_is_active(vma));
@@ -849,7 +873,10 @@ void i915_vma_destroy(struct i915_vma *vma)
if (i915_vma_is_closed(vma))
list_del(&vma->closed_link);
+ mutex_lock(&vm->mutex);
WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&vm->mutex);
+
__i915_vma_destroy(vma);
}
@@ -1020,64 +1047,44 @@ int i915_vma_unbind(struct i915_vma *vma)
{
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
+ might_sleep();
+
+ if (i915_vma_is_pinned(vma)) {
+ vma_print_allocator(vma, "is pinned");
+ return -EBUSY;
+ }
+
+ if (!drm_mm_node_allocated(&vma->node))
+ return 0;
/*
* First wait upon any activity as retiring the request may
* have side-effects such as unpinning or even unbinding this vma.
*/
- might_sleep();
if (i915_vma_is_active(vma)) {
struct i915_vma_active *active, *n;
- /*
- * When a closed VMA is retired, it is unbound - eek.
- * In order to prevent it from being recursively closed,
- * take a pin on the vma so that the second unbind is
- * aborted.
- *
- * Even more scary is that the retire callback may free
- * the object (last active vma). To prevent the explosion
- * we defer the actual object free to a worker that can
- * only proceed once it acquires the struct_mutex (which
- * we currently hold, therefore it cannot free this object
- * before we are finished).
- */
- __i915_vma_pin(vma);
-
- ret = i915_gem_active_retire(&vma->last_active,
- &vma->vm->i915->drm.struct_mutex);
+ ret = i915_gem_active_wait(&vma->last_active,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
- goto unpin;
+ return ret;
rbtree_postorder_for_each_entry_safe(active, n,
&vma->active, node) {
- ret = i915_gem_active_retire(&active->base,
- &vma->vm->i915->drm.struct_mutex);
+ ret = i915_gem_active_wait(&active->base,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
- goto unpin;
+ return ret;
}
- ret = i915_gem_active_retire(&vma->last_fence,
- &vma->vm->i915->drm.struct_mutex);
-unpin:
- __i915_vma_unpin(vma);
+ ret = i915_gem_active_wait(&vma->last_fence,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
return ret;
}
- GEM_BUG_ON(i915_vma_is_active(vma));
-
- if (i915_vma_is_pinned(vma)) {
- vma_print_allocator(vma, "is pinned");
- return -EBUSY;
- }
-
- if (!drm_mm_node_allocated(&vma->node))
- return 0;
if (i915_vma_is_map_and_fenceable(vma)) {
- mutex_lock(&vma->vm->mutex);
-
/*
* Check that we have flushed all writes through the GGTT
* before the unbind, other due to non-strict nature of those
@@ -1093,8 +1100,6 @@ int i915_vma_unbind(struct i915_vma *vma)
__i915_vma_iounmap(vma);
vma->flags &= ~I915_VMA_CAN_FENCE;
-
- mutex_unlock(&vma->vm->mutex);
}
GEM_BUG_ON(vma->fence);
GEM_BUG_ON(i915_vma_has_userfault(vma));
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index d1d0348a98d5..91dab9586b6a 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -277,11 +277,22 @@ void i915_vma_close(struct i915_vma *vma);
void i915_vma_reopen(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);
+static inline void ____i915_vma_pin(struct i915_vma *vma)
+{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ vma->flags++;
+ GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+}
+
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
+
static inline int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+__i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
+ lockdep_assert_held(&vma->vm->mutex);
+
BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
@@ -298,32 +309,42 @@ i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
return __i915_vma_do_pin(vma, size, alignment, flags);
}
-static inline int i915_vma_pin_count(const struct i915_vma *vma)
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- return vma->flags & I915_VMA_PIN_MASK;
+ int err;
+
+ mutex_lock(&vma->vm->mutex);
+ err = __i915_vma_pin(vma, size, alignment, flags);
+ mutex_unlock(&vma->vm->mutex);
+
+ return err;
}
-static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
+static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
- return i915_vma_pin_count(vma);
+ return READ_ONCE(vma->flags) & I915_VMA_PIN_MASK;
}
-static inline void __i915_vma_pin(struct i915_vma *vma)
+static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
{
- vma->flags++;
- GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+ return i915_vma_pin_count(vma);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ GEM_BUG_ON(!i915_vma_is_pinned(vma));
vma->flags--;
}
static inline void i915_vma_unpin(struct i915_vma *vma)
{
- GEM_BUG_ON(!i915_vma_is_pinned(vma));
+ mutex_lock(&vma->vm->mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
__i915_vma_unpin(vma);
+ mutex_unlock(&vma->vm->mutex);
}
static inline bool i915_vma_is_bound(const struct i915_vma *vma,
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 5f358e176a82..84d1d38a600f 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -490,6 +490,7 @@ void intel_engine_setup_common(struct intel_engine_cs *engine)
int intel_engine_create_scratch(struct intel_engine_cs *engine,
unsigned int size)
{
+ struct i915_ggtt *ggtt = &engine->i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int ret;
@@ -504,20 +505,21 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine,
return PTR_ERR(obj);
}
- vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_unref;
+ goto err_put_obj;
}
ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
if (ret)
- goto err_unref;
+ goto err_put_obj;
engine->scratch = vma;
return 0;
-err_unref:
+ mutex_unlock(&ggtt->vm.mutex);
+err_put_obj:
i915_gem_object_put(obj);
return ret;
}
@@ -558,6 +560,7 @@ static void cleanup_status_page(struct intel_engine_cs *engine)
static int init_status_page(struct intel_engine_cs *engine)
{
+ struct i915_ggtt *ggtt = &engine->i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
unsigned int flags;
@@ -574,7 +577,7 @@ static int init_status_page(struct intel_engine_cs *engine)
if (ret)
goto err;
- vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto err;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 6d7ce0b6b025..44a0d3010059 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1289,6 +1289,7 @@ static struct i915_vma *
alloc_context_vma(struct intel_engine_cs *engine)
{
struct drm_i915_private *i915 = engine->i915;
+ struct i915_ggtt *ggtt = &i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int err;
@@ -1339,7 +1340,7 @@ alloc_context_vma(struct intel_engine_cs *engine)
i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
}
- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
goto err_obj;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 0bc63fb03edd..7017adae7cc2 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -1514,7 +1514,7 @@ static int igt_gtt_insert(void *arg)
goto out;
}
track_vma_bind(vma);
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
}
--
2.18.0
More information about the Intel-gfx-trybot
mailing list