[PATCH 39/46] drm/i915: Move vma pinning under vm->mutex
Chris Wilson
chris at chris-wilson.co.uk
Sat Jul 14 17:15:50 UTC 2018
Use the local GTT lock (vm->mutex) for pinning the VMA (including
insertion and removal), as opposed to relying on the BKL struct_mutex.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem.c | 135 +++++++-----------
drivers/gpu/drm/i915/i915_gem_evict.c | 18 ++-
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 52 ++++---
drivers/gpu/drm/i915/i915_gem_gtt.c | 96 +++++++++----
drivers/gpu/drm/i915/i915_gem_render_state.c | 17 ++-
drivers/gpu/drm/i915/i915_gem_shrinker.c | 2 +
drivers/gpu/drm/i915/i915_gem_stolen.c | 8 +-
drivers/gpu/drm/i915/i915_gem_tiling.c | 30 ++--
drivers/gpu/drm/i915/i915_vma.c | 109 +++++++-------
drivers/gpu/drm/i915/i915_vma.h | 39 +++--
drivers/gpu/drm/i915/intel_engine_cs.c | 13 +-
drivers/gpu/drm/i915/intel_ringbuffer.c | 3 +-
.../gpu/drm/i915/selftests/i915_gem_evict.c | 76 +++++++---
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 2 +-
14 files changed, 346 insertions(+), 254 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index f918d1b1ef88..e03db9b29e69 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -67,17 +67,26 @@ static int
insert_mappable_node(struct i915_ggtt *ggtt,
struct drm_mm_node *node, u32 size)
{
+ int err;
+
memset(node, 0, sizeof(*node));
- return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
- size, 0, I915_COLOR_UNEVICTABLE,
- 0, ggtt->mappable_end,
- DRM_MM_INSERT_LOW);
+
+ mutex_lock(&ggtt->vm.mutex);
+ err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
+ size, 0, I915_COLOR_UNEVICTABLE,
+ 0, ggtt->mappable_end,
+ DRM_MM_INSERT_LOW);
+ mutex_unlock(&ggtt->vm.mutex);
+
+ return err;
}
static void
-remove_mappable_node(struct drm_mm_node *node)
+remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
{
+ mutex_lock(&ggtt->vm.mutex);
drm_mm_remove_node(node);
+ mutex_unlock(&ggtt->vm.mutex);
}
/* some bookkeeping */
@@ -445,7 +454,9 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
list_move_tail(&vma->obj_link, &still_in_list);
spin_unlock(&obj->vma.lock);
+ mutex_lock(&vma->vm->mutex);
ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
spin_lock(&obj->vma.lock);
}
@@ -1253,7 +1264,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
@@ -1461,7 +1472,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
@@ -2119,7 +2130,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
err_fence:
i915_vma_unpin_fence(vma);
err_unpin:
- __i915_vma_unpin(vma);
+ i915_vma_unpin(vma);
err_unlock:
mutex_unlock(&dev->struct_mutex);
err_rpm:
@@ -3605,11 +3616,13 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (obj->cache_level == cache_level)
return 0;
- /* Inspect the list of currently bound VMA and unbind any that would
+ /*
+ * Inspect the list of currently bound VMA and unbind any that would
* be invalid given the new cache-level. This is principally to
* catch the issue of the CS prefetch crossing page boundaries and
* reading an invalid PTE on older architectures.
*/
+ spin_lock(&obj->vma.lock);
restart:
list_for_each_entry(vma, &obj->vma.list, obj_link) {
if (!drm_mm_node_allocated(&vma->node))
@@ -3617,16 +3630,17 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
if (i915_vma_is_pinned(vma)) {
DRM_DEBUG("can not change the cache level of pinned objects\n");
- return -EBUSY;
+ ret = -EBUSY;
+ break;
}
- if (!i915_vma_is_closed(vma) &&
- i915_gem_valid_gtt_space(vma, cache_level))
- continue;
-
+ spin_unlock(&obj->vma.lock);
+ mutex_lock(&vma->vm->mutex);
ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ spin_lock(&obj->vma.lock);
if (ret)
- return ret;
+ break;
/* As unbinding may affect other elements in the
* obj->vma_list (due to side-effects from retiring
@@ -3634,70 +3648,9 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
*/
goto restart;
}
-
- /* We can reuse the existing drm_mm nodes but need to change the
- * cache-level on the PTE. We could simply unbind them all and
- * rebind with the correct cache-level on next use. However since
- * we already have a valid slot, dma mapping, pages etc, we may as
- * rewrite the PTE in the belief that doing so tramples upon less
- * state and so involves less work.
- */
- if (obj->bind_count) {
- /* Before we change the PTE, the GPU must not be accessing it.
- * If we wait upon the object, we know that all the bound
- * VMA are no longer active.
- */
- ret = i915_gem_object_wait(obj,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
- I915_WAIT_ALL,
- MAX_SCHEDULE_TIMEOUT,
- NULL);
- if (ret)
- return ret;
-
- if (!HAS_LLC(to_i915(obj->base.dev)) &&
- cache_level != I915_CACHE_NONE) {
- /* Access to snoopable pages through the GTT is
- * incoherent and on some machines causes a hard
- * lockup. Relinquish the CPU mmaping to force
- * userspace to refault in the pages and we can
- * then double check if the GTT mapping is still
- * valid for that pointer access.
- */
- i915_gem_release_mmap(obj);
-
- /* As we no longer need a fence for GTT access,
- * we can relinquish it now (and so prevent having
- * to steal a fence from someone else on the next
- * fence request). Note GPU activity would have
- * dropped the fence as all snoopable access is
- * supposed to be linear.
- */
- for_each_ggtt_vma(vma, obj) {
- ret = i915_vma_put_fence(vma);
- if (ret)
- return ret;
- }
- } else {
- /* We either have incoherent backing store and
- * so no GTT access or the architecture is fully
- * coherent. In such cases, existing GTT mmaps
- * ignore the cache bit in the PTE and we can
- * rewrite it without confusing the GPU or having
- * to force userspace to fault back in its mmaps.
- */
- }
-
- list_for_each_entry(vma, &obj->vma.list, obj_link) {
- if (!drm_mm_node_allocated(&vma->node))
- continue;
-
- ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
- if (ret)
- return ret;
- }
- }
+ spin_unlock(&obj->vma.lock);
+ if (ret)
+ return ret;
list_for_each_entry(vma, &obj->vma.list, obj_link)
vma->node.color = cache_level;
@@ -4006,8 +3959,6 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
if (flags & PIN_MAPPABLE &&
(!view || view->type == I915_GGTT_VIEW_NORMAL)) {
/* If the required space is larger than the available
@@ -4044,14 +3995,18 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
if (unlikely(IS_ERR(vma)))
return vma;
+ mutex_lock(&vm->mutex);
+
if (i915_vma_misplaced(vma, size, alignment, flags)) {
if (flags & PIN_NONBLOCK) {
+ ret = -ENOSPC;
+
if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
- return ERR_PTR(-ENOSPC);
+ goto err_unlock;
if (flags & PIN_MAPPABLE &&
vma->fence_size > dev_priv->ggtt.mappable_end / 2)
- return ERR_PTR(-ENOSPC);
+ goto err_unlock;
}
WARN(i915_vma_is_pinned(vma),
@@ -4061,16 +4016,22 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
i915_ggtt_offset(vma), alignment,
!!(flags & PIN_MAPPABLE),
i915_vma_is_map_and_fenceable(vma));
+
ret = i915_vma_unbind(vma);
if (ret)
- return ERR_PTR(ret);
+ goto err_unlock;
}
- ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
+ ret = __i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
if (ret)
- return ERR_PTR(ret);
+ goto err_unlock;
+ mutex_unlock(&vm->mutex);
return vma;
+
+err_unlock:
+ mutex_unlock(&vm->mutex);
+ return ERR_PTR(ret);
}
static __always_inline unsigned int __busy_read_flag(unsigned int id)
@@ -4996,7 +4957,9 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
* from the GTT to prevent such accidents and reclaim the
* space.
*/
+ mutex_lock(&state->vm->mutex);
err = i915_vma_unbind(state);
+ mutex_unlock(&state->vm->mutex);
if (err)
goto err_active;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 11742f8701d7..8f4609731eb6 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -63,7 +63,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
* the hopes that we can then remove contexts and the like only
* bound by their active reference.
*/
- err = i915_gem_switch_to_kernel_context(i915, GFP_KERNEL);
+ err = i915_gem_switch_to_kernel_context(i915, GFP_NOWAIT);
if (err)
return err;
@@ -133,7 +133,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
struct i915_vma *active;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
@@ -224,7 +224,9 @@ i915_gem_evict_something(struct i915_address_space *vm,
if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
return -EBUSY;
+ mutex_unlock(&vm->mutex);
ret = ggtt_flush(dev_priv);
+ mutex_lock(&vm->mutex);
if (ret)
return ret;
@@ -248,7 +250,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
*/
list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
if (drm_mm_scan_remove_block(&scan, &vma->node))
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
else
list_del(&vma->evict_link);
}
@@ -292,7 +294,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
bool check_color;
int ret = 0;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
@@ -372,7 +374,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
* unbinding one vma from freeing (by dropping its active
* reference) another in our eviction list.
*/
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
@@ -403,7 +405,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
struct i915_vma *vma, *next;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict_vm(vm);
/* Switch back to the default context in order to unpin
@@ -412,7 +414,9 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
* switch otherwise is ineffective.
*/
if (i915_is_ggtt(vm)) {
+ mutex_unlock(&vm->mutex);
ret = ggtt_flush(vm->i915);
+ mutex_lock(&vm->mutex);
if (ret)
return ret;
}
@@ -422,7 +426,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
if (i915_vma_is_pinned(vma))
continue;
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 2021de82ba50..32cbdfd4f16d 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -405,7 +405,7 @@ eb_pin_vma(struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
pin_flags |= PIN_GLOBAL;
- if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
+ if (unlikely(__i915_vma_pin(vma, 0, 0, pin_flags)))
return false;
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
@@ -617,9 +617,9 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
}
- err = i915_vma_pin(vma,
- entry->pad_to_size, entry->alignment,
- pin_flags);
+ err = __i915_vma_pin(vma,
+ entry->pad_to_size, entry->alignment,
+ pin_flags);
if (err)
return err;
@@ -631,7 +631,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
err = i915_vma_reserve_fence(vma);
if (unlikely(err)) {
- i915_vma_unpin(vma);
+ __i915_vma_unpin(vma);
return err;
}
@@ -762,6 +762,7 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
batch = eb_batch_index(eb);
+ mutex_lock(&eb->vm->mutex);
for (i = 0; i < eb->buffer_count; i++) {
u32 handle = eb->exec[i].handle;
struct i915_lut_handle *lut;
@@ -771,6 +772,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
if (likely(vma))
goto add_vma;
+ mutex_unlock(&eb->vm->mutex);
+
obj = i915_gem_object_lookup(eb->file, handle);
if (unlikely(!obj)) {
err = -ENOENT;
@@ -795,6 +798,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
goto err_obj;
}
+ mutex_lock(&eb->vm->mutex);
+
/* transfer ref to ctx */
if (!vma->open_count++)
i915_vma_reopen(vma);
@@ -805,8 +810,10 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
add_vma:
err = eb_add_vma(eb, i, batch, vma);
- if (unlikely(err))
+ if (unlikely(err)) {
+ mutex_unlock(&eb->vm->mutex);
goto err_vma;
+ }
GEM_BUG_ON(vma != eb->vma[i]);
GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
@@ -815,7 +822,9 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
}
eb->args->flags |= __EXEC_VALIDATED;
- return eb_reserve(eb);
+ err = eb_reserve(eb);
+ mutex_unlock(&eb->vm->mutex);
+ return err;
err_obj:
i915_gem_object_put(obj);
@@ -849,6 +858,7 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
const unsigned int count = eb->buffer_count;
unsigned int i;
+ mutex_lock(&eb->vm->mutex);
for (i = 0; i < count; i++) {
struct i915_vma *vma = eb->vma[i];
unsigned int flags = eb->flags[i];
@@ -860,12 +870,20 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
vma->exec_flags = NULL;
eb->vma[i] = NULL;
+ if (unlikely(vma->vm != eb->vm))
+ mutex_lock_nested(&vma->vm->mutex,
+ SINGLE_DEPTH_NESTING);
+
if (flags & __EXEC_OBJECT_HAS_PIN)
__eb_unreserve_vma(vma, flags);
if (flags & __EXEC_OBJECT_HAS_REF)
i915_vma_put(vma);
+
+ if (unlikely(vma->vm != eb->vm))
+ mutex_unlock(&vma->vm->mutex);
}
+ mutex_unlock(&eb->vm->mutex);
}
static void eb_reset_vmas(const struct i915_execbuffer *eb)
@@ -1364,8 +1382,14 @@ eb_relocate_entry(struct i915_execbuffer *eb,
*/
if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
IS_GEN6(eb->i915)) {
+ reloc_cache_reset(&eb->reloc_cache);
+
+ GEM_BUG_ON(!i915_vma_is_ggtt(target));
+ mutex_lock(&target->vm->mutex);
err = i915_vma_bind(target, target->obj->cache_level,
PIN_GLOBAL);
+ mutex_unlock(&target->vm->mutex);
+
if (WARN_ONCE(err,
"Unexpected failure to bind target VMA!"))
return err;
@@ -1829,22 +1853,14 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
}
for (i = 0; i < count; i++) {
- unsigned int flags = eb->flags[i];
- struct i915_vma *vma = eb->vma[i];
-
- err = i915_vma_move_to_active(vma, eb->request, flags);
+ err = i915_vma_move_to_active(eb->vma[i],
+ eb->request,
+ eb->flags[i]);
if (unlikely(err)) {
i915_request_skip(eb->request, err);
return err;
}
-
- __eb_unreserve_vma(vma, flags);
- vma->exec_flags = NULL;
-
- if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
- i915_vma_put(vma);
}
- eb->exec = NULL;
/* Unconditionally flush any chipset caches (for streaming writes). */
i915_gem_chipset_flush(eb->i915);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 3a5860488282..b56a97281769 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1416,17 +1416,24 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
if (pt == vm->scratch_pt) {
pd->used_pdes++;
+ mutex_unlock(&vm->mutex);
pt = alloc_pt(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pt)) {
pd->used_pdes--;
goto unwind;
}
- if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
- gen8_initialize_pt(vm, pt);
+ if (pd->page_table[pde] == vm->scratch_pt) {
+ if (count < GEN8_PTES ||
+ intel_vgpu_active(vm->i915))
+ gen8_initialize_pt(vm, pt);
- gen8_ppgtt_set_pde(vm, pd, pt, pde);
- GEM_BUG_ON(pd->used_pdes > I915_PDES);
+ gen8_ppgtt_set_pde(vm, pd, pt, pde);
+ GEM_BUG_ON(pd->used_pdes > I915_PDES);
+ } else {
+ free_pt(vm, pt);
+ }
}
pt->used_ptes += count;
@@ -1451,17 +1458,23 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
if (pd == vm->scratch_pd) {
pdp->used_pdpes++;
+ mutex_unlock(&vm->mutex);
pd = alloc_pd(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pd)) {
pdp->used_pdpes--;
goto unwind;
}
- gen8_initialize_pd(vm, pd);
- gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
- GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
+ if (pdp->page_directory[pdpe] == vm->scratch_pd) {
+ gen8_initialize_pd(vm, pd);
+ gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
+ GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
- mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
+ mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
+ } else {
+ free_pd(vm, pd);
+ }
}
ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
@@ -1502,12 +1515,18 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+ mutex_unlock(&vm->mutex);
pdp = alloc_pdp(vm);
+ mutex_lock(&vm->mutex);
if (IS_ERR(pdp))
goto unwind;
- gen8_initialize_pdp(vm, pdp);
- gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+ if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+ gen8_initialize_pdp(vm, pdp);
+ gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+ } else {
+ free_pdp(vm, pdp);
+ }
}
ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
@@ -3026,8 +3045,10 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
mutex_lock(&dev_priv->drm.struct_mutex);
i915_gem_fini_aliasing_ppgtt(dev_priv);
+ mutex_lock(&ggtt->vm.mutex);
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&ggtt->vm.mutex);
if (drm_mm_node_allocated(&ggtt->error_capture))
drm_mm_remove_node(&ggtt->error_capture);
@@ -3734,8 +3755,6 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
/* clflush objects bound into the GGTT and rebind them. */
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
- struct drm_i915_gem_object *obj = vma->obj;
-
if (!(vma->flags & I915_VMA_GLOBAL_BIND))
continue;
@@ -3743,10 +3762,8 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
continue;
WARN_ON(i915_vma_bind(vma,
- obj ? obj->cache_level : 0,
+ vma->obj ? vma->obj->cache_level : 0,
PIN_UPDATE));
- if (obj)
- WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
}
ggtt->vm.closed = false;
@@ -3754,6 +3771,13 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
mutex_unlock(&ggtt->vm.mutex);
+ list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
+ if (!vma->obj || !(vma->flags & I915_VMA_GLOBAL_BIND))
+ continue;
+
+ WARN_ON(i915_gem_object_set_to_gtt_domain(vma->obj, false));
+ }
+
if (INTEL_GEN(dev_priv) >= 8) {
struct intel_ppat *ppat = &dev_priv->ppat;
@@ -3905,9 +3929,10 @@ intel_partial_pages(const struct i915_ggtt_view *view,
static int
i915_get_ggtt_vma_pages(struct i915_vma *vma)
{
- int ret;
+ struct sg_table *pages;
- /* The vma->pages are only valid within the lifespan of the borrowed
+ /*
+ * The vma->pages are only valid within the lifespan of the borrowed
* obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
* must be the vma->pages. A simple rule is that vma->pages must only
* be accessed when the obj->mm.pages are pinned.
@@ -3919,27 +3944,35 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
GEM_BUG_ON(vma->ggtt_view.type);
/* fall through */
case I915_GGTT_VIEW_NORMAL:
- vma->pages = vma->obj->mm.pages;
- return 0;
+ pages = vma->obj->mm.pages;
+ goto out;
+
+ case I915_GGTT_VIEW_ROTATED:
+ case I915_GGTT_VIEW_PARTIAL:
+ break;
+ }
+ GEM_BUG_ON(!i915_vma_is_pinned(vma));
+ mutex_unlock(&vma->vm->mutex);
+ switch ((int)vma->ggtt_view.type) {
case I915_GGTT_VIEW_ROTATED:
- vma->pages =
- intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
+ pages = intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
break;
case I915_GGTT_VIEW_PARTIAL:
- vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
+ pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
+ break;
+ default:
+ pages = ERR_PTR(-ENODEV);
break;
}
+ mutex_lock(&vma->vm->mutex);
+ if (IS_ERR(pages))
+ return PTR_ERR(pages);
- ret = 0;
- if (unlikely(IS_ERR(vma->pages))) {
- ret = PTR_ERR(vma->pages);
- vma->pages = NULL;
- DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
- vma->ggtt_view.type, ret);
- }
- return ret;
+out:
+ vma->pages = pages;
+ return 0;
}
/**
@@ -3974,6 +4007,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm,
{
int err;
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
@@ -4067,7 +4101,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
u64 offset;
int err;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(alignment && !is_power_of_2(alignment));
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 90baf9086d0a..904581fdbde7 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -179,6 +179,7 @@ static int render_state_setup(struct intel_render_state *so,
int i915_gem_render_state_emit(struct i915_request *rq)
{
+ struct i915_ggtt *ggtt = &rq->i915->ggtt;
struct intel_engine_cs *engine = rq->engine;
struct intel_render_state so = {}; /* keep the compiler happy */
int err;
@@ -194,15 +195,19 @@ int i915_gem_render_state_emit(struct i915_request *rq)
if (IS_ERR(so.obj))
return PTR_ERR(so.obj);
- so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.vm, NULL);
+ so.vma = i915_vma_instance(so.obj, &ggtt->vm, NULL);
if (IS_ERR(so.vma)) {
err = PTR_ERR(so.vma);
goto err_obj;
}
- err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+ mutex_lock(&ggtt->vm.mutex);
+
+ err = __i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
if (err)
- goto err_vma;
+ goto err_vma_locked;
+
+ mutex_unlock(&ggtt->vm.mutex);
err = render_state_setup(&so, rq->i915);
if (err)
@@ -224,9 +229,11 @@ int i915_gem_render_state_emit(struct i915_request *rq)
err = i915_vma_move_to_active(so.vma, rq, 0);
err_unpin:
- i915_vma_unpin(so.vma);
-err_vma:
+ mutex_lock(&ggtt->vm.mutex);
+ __i915_vma_unpin(so.vma);
+err_vma_locked:
i915_vma_close(so.vma);
+ mutex_unlock(&ggtt->vm.mutex);
err_obj:
__i915_gem_object_release_unless_active(so.obj);
return err;
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index bebcc8ea5725..99f3f002358a 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -485,6 +485,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
intel_runtime_pm_put(i915);
/* We also want to clear any cached iomaps as they wrap vmap */
+ mutex_lock(&i915->ggtt.vm.mutex);
list_for_each_entry_safe(vma, next,
&i915->ggtt.vm.bound_list, vm_link) {
unsigned long count = vma->node.size >> PAGE_SHIFT;
@@ -495,6 +496,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
if (i915_vma_unbind(vma) == 0)
freed_pages += count;
}
+ mutex_unlock(&i915->ggtt.vm.mutex);
out:
shrinker_unlock(i915, unlock);
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index f1e47414523a..0e760945b15c 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -637,8 +637,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
if (!drm_mm_initialized(&dev_priv->mm.stolen))
return NULL;
- lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
&stolen_offset, >t_offset, &size);
@@ -685,7 +683,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
goto err_pages;
}
- /* To simplify the initialisation sequence between KMS and GTT,
+ mutex_lock(&vma->vm->mutex);
+
+ /*
+ * To simplify the initialisation sequence between KMS and GTT,
* we allow construction of the stolen object prior to
* setting up the GTT space. The actual reservation will occur
* later.
@@ -704,7 +705,6 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
vma->flags |= I915_VMA_GLOBAL_BIND;
__i915_vma_set_map_and_fenceable(vma);
- mutex_lock(&ggtt->vm.mutex);
list_add_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index d9dc9df523b5..8292492b1b86 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -199,22 +199,25 @@ static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
int tiling_mode, unsigned int stride)
{
+ struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
struct i915_vma *vma;
int ret;
if (tiling_mode == I915_TILING_NONE)
return 0;
+ mutex_lock(&ggtt->vm.mutex);
for_each_ggtt_vma(vma, obj) {
if (i915_vma_fence_prepare(vma, tiling_mode, stride))
continue;
ret = i915_vma_unbind(vma);
if (ret)
- return ret;
+ break;
}
+ mutex_unlock(&ggtt->vm.mutex);
- return 0;
+ return ret;
}
int
@@ -251,16 +254,20 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
* whilst executing a fenced command for an untiled object.
*/
- err = i915_gem_object_fence_prepare(obj, tiling, stride);
- if (err)
- return err;
-
i915_gem_object_lock(obj);
+
if (i915_gem_object_is_framebuffer(obj)) {
- i915_gem_object_unlock(obj);
- return -EBUSY;
+ err = -EBUSY;
+ goto unlock;
}
+ err = i915_gem_object_fence_prepare(obj, tiling, stride);
+ if (err)
+ goto unlock;
+
+ /* Force the fence to be reacquired for GTT access */
+ i915_gem_release_mmap(obj);
+
/* If the memory has unknown (i.e. varying) swizzling, we pin the
* pages to prevent them being swapped out and causing corruption
* due to the change in swizzling.
@@ -296,9 +303,6 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
obj->tiling_and_stride = tiling | stride;
i915_gem_object_unlock(obj);
- /* Force the fence to be reacquired for GTT access */
- i915_gem_release_mmap(obj);
-
/* Try to preallocate memory required to save swizzling on put-pages */
if (i915_gem_object_needs_bit17_swizzle(obj)) {
if (!obj->bit_17) {
@@ -311,6 +315,10 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
}
return 0;
+
+unlock:
+ i915_gem_object_unlock(obj);
+ return err;
}
/**
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 790a260dc31b..12d79aa6911c 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -274,7 +274,7 @@ vma_lookup(struct drm_i915_gem_object *obj,
* Once created, the VMA is kept until either the object is freed, or the
* address space is closed.
*
- * Must be called with struct_mutex held.
+ * Must be called with vm->mutex held.
*
* Returns the vma, or an error pointer.
*/
@@ -316,6 +316,7 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
u32 vma_flags;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->size > vma->node.size);
@@ -359,8 +360,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
/* Access through the GTT requires the device to be awake. */
assert_rpm_wakelock_held(vma->vm->i915);
+ mutex_lock(&vma->vm->mutex);
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
err = -ENODEV;
goto err;
@@ -382,13 +383,12 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
vma->iomap = ptr;
}
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma);
if (err)
goto err_unpin;
- mutex_lock(&vma->vm->mutex);
i915_vma_set_ggtt_write(vma);
mutex_unlock(&vma->vm->mutex);
return ptr;
@@ -396,6 +396,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
err_unpin:
__i915_vma_unpin(vma);
err:
+ mutex_unlock(&vma->vm->mutex);
return IO_ERR_PTR(err);
}
@@ -564,6 +565,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
u64 start, end;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(i915_vma_is_closed(vma));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
@@ -590,7 +593,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
- /* If binding the object/GGTT view requires more space than the entire
+ /*
+ * If binding the object/GGTT view requires more space than the entire
* aperture has, reject it early before evicting everything in a vain
* attempt to find space.
*/
@@ -602,11 +606,28 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
}
if (vma->obj) {
- ret = i915_gem_object_pin_pages(vma->obj);
+ struct drm_i915_gem_object *obj = vma->obj;
+
+ mutex_unlock(&vma->vm->mutex);
+ ret = i915_gem_object_pin_pages(obj);
+ mutex_lock(&vma->vm->mutex);
if (ret)
return ret;
- cache_level = vma->obj->cache_level;
+ /*
+ * Check that someone else didn't complete the job on our
+ * behalf while we dropped the lock.
+ */
+ if (drm_mm_node_allocated(&vma->node)) {
+ i915_gem_object_unpin_pages(obj);
+
+ if (i915_vma_misplaced(vma, size, alignment, flags))
+ return -EAGAIN;
+
+ return 0;
+ }
+
+ cache_level = obj->cache_level;
} else {
cache_level = 0;
}
@@ -676,9 +697,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
- mutex_lock(&vma->vm->mutex);
+ lockdep_assert_held(&vma->vm->mutex);
list_add_tail(&vma->vm_link, &vma->vm->bound_list);
- mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
@@ -706,15 +726,15 @@ i915_vma_remove(struct i915_vma *vma)
{
struct drm_i915_private *i915 = vma->vm->i915;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
vma->ops->clear_pages(vma);
- mutex_lock(&vma->vm->mutex);
drm_mm_remove_node(&vma->node);
list_del(&vma->vm_link);
- mutex_unlock(&vma->vm->mutex);
/*
* Since the unbound list is global, only move to that list if
@@ -744,7 +764,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
const unsigned int bound = vma->flags;
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
@@ -847,6 +867,8 @@ static void __i915_vma_destroy(struct i915_vma *vma)
void i915_vma_destroy(struct i915_vma *vma)
{
+ struct i915_address_space *vm = vma->vm;
+
lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
GEM_BUG_ON(i915_vma_is_active(vma));
@@ -855,7 +877,10 @@ void i915_vma_destroy(struct i915_vma *vma)
if (i915_vma_is_closed(vma))
list_del(&vma->closed_link);
+ mutex_lock(&vm->mutex);
WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&vm->mutex);
+
__i915_vma_destroy(vma);
}
@@ -1048,64 +1073,44 @@ int i915_vma_unbind(struct i915_vma *vma)
{
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
+ might_sleep();
+
+ if (i915_vma_is_pinned(vma)) {
+ vma_print_allocator(vma, "is pinned");
+ return -EBUSY;
+ }
+
+ if (!drm_mm_node_allocated(&vma->node))
+ return 0;
/*
* First wait upon any activity as retiring the request may
* have side-effects such as unpinning or even unbinding this vma.
*/
- might_sleep();
if (i915_vma_is_active(vma)) {
struct i915_vma_active *active, *n;
- /*
- * When a closed VMA is retired, it is unbound - eek.
- * In order to prevent it from being recursively closed,
- * take a pin on the vma so that the second unbind is
- * aborted.
- *
- * Even more scary is that the retire callback may free
- * the object (last active vma). To prevent the explosion
- * we defer the actual object free to a worker that can
- * only proceed once it acquires the struct_mutex (which
- * we currently hold, therefore it cannot free this object
- * before we are finished).
- */
- __i915_vma_pin(vma);
-
- ret = i915_gem_active_retire(&vma->last_active,
- &vma->vm->i915->drm.struct_mutex);
+ ret = i915_gem_active_wait(&vma->last_active,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
- goto unpin;
+ return ret;
rbtree_postorder_for_each_entry_safe(active, n,
&vma->active, node) {
- ret = i915_gem_active_retire(&active->base,
- &vma->vm->i915->drm.struct_mutex);
+ ret = i915_gem_active_wait(&active->base,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
- goto unpin;
+ return ret;
}
- ret = i915_gem_active_retire(&vma->last_fence,
- &vma->vm->i915->drm.struct_mutex);
-unpin:
- __i915_vma_unpin(vma);
+ ret = i915_gem_active_wait(&vma->last_fence,
+ I915_WAIT_INTERRUPTIBLE);
if (ret)
return ret;
}
- GEM_BUG_ON(i915_vma_is_active(vma));
-
- if (i915_vma_is_pinned(vma)) {
- vma_print_allocator(vma, "is pinned");
- return -EBUSY;
- }
-
- if (!drm_mm_node_allocated(&vma->node))
- return 0;
if (i915_vma_is_map_and_fenceable(vma)) {
- mutex_lock(&vma->vm->mutex);
-
/*
* Check that we have flushed all writes through the GGTT
* before the unbind, other due to non-strict nature of those
@@ -1121,8 +1126,6 @@ int i915_vma_unbind(struct i915_vma *vma)
__i915_vma_iounmap(vma);
vma->flags &= ~I915_VMA_CAN_FENCE;
-
- mutex_unlock(&vma->vm->mutex);
}
GEM_BUG_ON(vma->fence);
GEM_BUG_ON(i915_vma_has_userfault(vma));
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 4c84a70eb7f3..6a12ea7c87c6 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -280,11 +280,22 @@ void i915_vma_close(struct i915_vma *vma);
void i915_vma_reopen(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);
+static inline void ____i915_vma_pin(struct i915_vma *vma)
+{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ vma->flags++;
+ GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+}
+
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
+
static inline int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+__i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
+ lockdep_assert_held(&vma->vm->mutex);
+
BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
@@ -301,32 +312,42 @@ i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
return __i915_vma_do_pin(vma, size, alignment, flags);
}
-static inline int i915_vma_pin_count(const struct i915_vma *vma)
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- return vma->flags & I915_VMA_PIN_MASK;
+ int err;
+
+ mutex_lock(&vma->vm->mutex);
+ err = __i915_vma_pin(vma, size, alignment, flags);
+ mutex_unlock(&vma->vm->mutex);
+
+ return err;
}
-static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
+static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
- return i915_vma_pin_count(vma);
+ return READ_ONCE(vma->flags) & I915_VMA_PIN_MASK;
}
-static inline void __i915_vma_pin(struct i915_vma *vma)
+static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
{
- vma->flags++;
- GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+ return i915_vma_pin_count(vma);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ GEM_BUG_ON(!i915_vma_is_pinned(vma));
vma->flags--;
}
static inline void i915_vma_unpin(struct i915_vma *vma)
{
- GEM_BUG_ON(!i915_vma_is_pinned(vma));
+ mutex_lock(&vma->vm->mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
__i915_vma_unpin(vma);
+ mutex_unlock(&vma->vm->mutex);
}
static inline bool i915_vma_is_bound(const struct i915_vma *vma,
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 816619d300b4..8a476f446da9 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -493,6 +493,7 @@ void intel_engine_setup_common(struct intel_engine_cs *engine)
int intel_engine_create_scratch(struct intel_engine_cs *engine,
unsigned int size)
{
+ struct i915_ggtt *ggtt = &engine->i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int ret;
@@ -507,20 +508,21 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine,
return PTR_ERR(obj);
}
- vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_unref;
+ goto err_put_obj;
}
ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
if (ret)
- goto err_unref;
+ goto err_put_obj;
engine->scratch = vma;
return 0;
-err_unref:
+ mutex_unlock(&ggtt->vm.mutex);
+err_put_obj:
i915_gem_object_put(obj);
return ret;
}
@@ -561,6 +563,7 @@ static void cleanup_status_page(struct intel_engine_cs *engine)
static int init_status_page(struct intel_engine_cs *engine)
{
+ struct i915_ggtt *ggtt = &engine->i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
unsigned int flags;
@@ -577,7 +580,7 @@ static int init_status_page(struct intel_engine_cs *engine)
if (ret)
goto err;
- vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto err;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 6077f7575677..64c6d80770a0 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1287,6 +1287,7 @@ static struct i915_vma *
alloc_context_vma(struct intel_engine_cs *engine)
{
struct drm_i915_private *i915 = engine->i915;
+ struct i915_ggtt *ggtt = &i915->ggtt;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int err;
@@ -1337,7 +1338,7 @@ alloc_context_vma(struct intel_engine_cs *engine)
i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
}
- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
goto err_obj;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index dc5e77af27d2..1b9dd1aa5e93 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -70,10 +70,9 @@ static void unpin_ggtt(struct drm_i915_private *i915)
struct i915_ggtt *ggtt = &i915->ggtt;
struct i915_vma *vma;
- mutex_lock(&ggtt->vm.mutex);
+ lockdep_assert_held(&ggtt->vm.mutex);
list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
i915_vma_unpin(vma);
- mutex_unlock(&ggtt->vm.mutex);
}
static void cleanup_objects(struct drm_i915_private *i915)
@@ -105,6 +104,8 @@ static int igt_evict_something(void *arg)
if (err)
goto cleanup;
+ mutex_lock(&ggtt->vm.mutex);
+
/* Everything is pinned, nothing should happen */
err = i915_gem_evict_something(&ggtt->vm,
I915_GTT_PAGE_SIZE, 0, 0,
@@ -113,7 +114,7 @@ static int igt_evict_something(void *arg)
if (err != -ENOSPC) {
pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
err);
- goto cleanup;
+ goto unlock;
}
unpin_ggtt(i915);
@@ -126,9 +127,11 @@ static int igt_evict_something(void *arg)
if (err) {
pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
err);
- goto cleanup;
+ goto unlock;
}
+unlock:
+ mutex_unlock(&ggtt->vm.mutex);
cleanup:
cleanup_objects(i915);
return err;
@@ -183,12 +186,14 @@ static int igt_evict_for_vma(void *arg)
if (err)
goto cleanup;
+ mutex_lock(&ggtt->vm.mutex);
+
/* Everything is pinned, nothing should happen */
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
if (err != -ENOSPC) {
pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n",
err);
- goto cleanup;
+ goto unlock;
}
unpin_ggtt(i915);
@@ -198,9 +203,11 @@ static int igt_evict_for_vma(void *arg)
if (err) {
pr_err("i915_gem_evict_for_node returned err=%d\n",
err);
- goto cleanup;
+ goto unlock;
}
+unlock:
+ mutex_unlock(&ggtt->vm.mutex);
cleanup:
cleanup_objects(i915);
return err;
@@ -265,12 +272,15 @@ static int igt_evict_for_cache_color(void *arg)
goto cleanup;
}
+ mutex_lock(&ggtt->vm.mutex);
+
i915_vma_unpin(vma);
/* Remove just the second vma */
err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
if (err) {
pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);
+ mutex_unlock(&ggtt->vm.mutex);
goto cleanup;
}
@@ -283,13 +293,18 @@ static int igt_evict_for_cache_color(void *arg)
if (!err) {
pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);
err = -EINVAL;
+ mutex_unlock(&ggtt->vm.mutex);
goto cleanup;
}
+ mutex_unlock(&ggtt->vm.mutex);
+
err = 0;
cleanup:
+ mutex_lock(&ggtt->vm.mutex);
unpin_ggtt(i915);
+ mutex_lock(&ggtt->vm.mutex);
cleanup_objects(i915);
ggtt->vm.mm.color_adjust = NULL;
return err;
@@ -307,12 +322,14 @@ static int igt_evict_vm(void *arg)
if (err)
goto cleanup;
+ mutex_lock(&ggtt->vm.mutex);
+
/* Everything is pinned, nothing should happen */
err = i915_gem_evict_vm(&ggtt->vm);
if (err) {
pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
err);
- goto cleanup;
+ goto unlock;
}
unpin_ggtt(i915);
@@ -321,9 +338,11 @@ static int igt_evict_vm(void *arg)
if (err) {
pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
err);
- goto cleanup;
+ goto unlock;
}
+unlock:
+ mutex_unlock(&ggtt->vm.mutex);
cleanup:
cleanup_objects(i915);
return err;
@@ -333,6 +352,7 @@ static int igt_evict_contexts(void *arg)
{
const u64 PRETEND_GGTT_SIZE = 16ull << 20;
struct drm_i915_private *i915 = arg;
+ struct i915_ggtt *ggtt = &i915->ggtt;
struct intel_engine_cs *engine;
enum intel_engine_id id;
struct reserved {
@@ -357,33 +377,39 @@ static int igt_evict_contexts(void *arg)
if (!USES_FULL_PPGTT(i915))
return 0;
- mutex_lock(&i915->drm.struct_mutex);
intel_runtime_pm_get(i915);
/* Reserve a block so that we know we have enough to fit a few rq */
memset(&hole, 0, sizeof(hole));
- err = i915_gem_gtt_insert(&i915->ggtt.vm, &hole,
+ mutex_lock(&ggtt->vm.mutex);
+ err = i915_gem_gtt_insert(&ggtt->vm, &hole,
PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE,
- 0, i915->ggtt.vm.total,
+ 0, ggtt->vm.total,
PIN_NOEVICT);
+ mutex_unlock(&ggtt->vm.mutex);
if (err)
- goto out_locked;
+ goto out;
/* Make the GGTT appear small by filling it with unevictable nodes */
count = 0;
do {
struct reserved *r;
+ int noerr;
r = kcalloc(1, sizeof(*r), GFP_KERNEL);
if (!r) {
err = -ENOMEM;
- goto out_locked;
+ goto out;
}
- if (i915_gem_gtt_insert(&i915->ggtt.vm, &r->node,
- 1ul << 20, 0, I915_COLOR_UNEVICTABLE,
- 0, i915->ggtt.vm.total,
- PIN_NOEVICT)) {
+ mutex_lock(&ggtt->vm.mutex);
+ noerr = i915_gem_gtt_insert(&ggtt->vm, &r->node,
+ 1ul << 20, 0,
+ I915_COLOR_UNEVICTABLE,
+ 0, ggtt->vm.total,
+ PIN_NOEVICT);
+ mutex_unlock(&ggtt->vm.mutex);
+ if (noerr) {
kfree(r);
break;
}
@@ -393,8 +419,10 @@ static int igt_evict_contexts(void *arg)
count++;
} while (1);
+
+ mutex_lock(&ggtt->vm.mutex);
drm_mm_remove_node(&hole);
- mutex_unlock(&i915->drm.struct_mutex);
+ mutex_unlock(&ggtt->vm.mutex);
pr_info("Filled GGTT with %lu 1MiB nodes\n", count);
/* Overfill the GGTT with context objects and so try to evict one. */
@@ -403,8 +431,10 @@ static int igt_evict_contexts(void *arg)
struct drm_file *file;
file = mock_file(i915);
- if (IS_ERR(file))
- return PTR_ERR(file);
+ if (IS_ERR(file)) {
+ err = PTR_ERR(file);
+ goto out;
+ }
count = 0;
mutex_lock(&i915->drm.struct_mutex);
@@ -455,8 +485,8 @@ static int igt_evict_contexts(void *arg)
break;
}
- mutex_lock(&i915->drm.struct_mutex);
-out_locked:
+out:
+ mutex_lock(&ggtt->vm.mutex);
while (reserved) {
struct reserved *next = reserved->next;
@@ -467,8 +497,8 @@ static int igt_evict_contexts(void *arg)
}
if (drm_mm_node_allocated(&hole))
drm_mm_remove_node(&hole);
+ mutex_unlock(&ggtt->vm.mutex);
intel_runtime_pm_put(i915);
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index c6e9800d2d83..2b4819048fae 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -1544,7 +1544,7 @@ static int igt_gtt_insert(void *arg)
goto out;
}
track_vma_bind(vma);
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
}
--
2.18.0
More information about the Intel-gfx-trybot
mailing list