[Intel-gfx] [PATCH 34/39] drm/i915: Use vm->mutex for serialising GTT insertion
Chris Wilson
chris at chris-wilson.co.uk
Fri Jun 14 07:10:18 UTC 2019
Serialising insertion into each of the global GTT and ppGTT accounts for
a large chunk of the current struct_mutex serialisation requireemnts.
(Note that it is not just the drm_mm / gtt management itself being
serialised, but the pin count and various flags.) Previously, the main
blocker for replacing this mutex was the reset handing, but with the
advent of "lockless" resets, we can freely take the vm->mutex and block
waiting for the GPU (without fear of deadlock if the GPU hangs). We also
proscribe allocations underneath vm->mutex, allowing us to take the
mutex inside the shrinker, avoiding the recursive struct_mutex we
previously used.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
.../gpu/drm/i915/gem/i915_gem_client_blt.c | 7 +-
drivers/gpu/drm/i915/gem/i915_gem_context.c | 5 +-
drivers/gpu/drm/i915/gem/i915_gem_domain.c | 33 +++---
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 5 +-
drivers/gpu/drm/i915/gem/i915_gem_mman.c | 27 ++---
drivers/gpu/drm/i915/gem/i915_gem_object.h | 3 +-
.../gpu/drm/i915/gem/i915_gem_object_types.h | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 93 +--------------
drivers/gpu/drm/i915/gem/i915_gem_tiling.c | 60 +++++-----
drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 28 +----
drivers/gpu/drm/i915/gt/intel_context.c | 5 +-
drivers/gpu/drm/i915/gt/intel_ringbuffer.c | 4 +-
drivers/gpu/drm/i915/gvt/aperture_gm.c | 12 +-
drivers/gpu/drm/i915/i915_debugfs.c | 10 +-
drivers/gpu/drm/i915/i915_gem.c | 106 ++++++++++--------
drivers/gpu/drm/i915/i915_gem_evict.c | 28 +----
drivers/gpu/drm/i915/i915_gem_fence_reg.c | 2 +-
drivers/gpu/drm/i915/i915_gem_gtt.c | 76 +------------
drivers/gpu/drm/i915/i915_gem_gtt.h | 3 -
drivers/gpu/drm/i915/i915_perf.c | 20 +---
drivers/gpu/drm/i915/i915_vma.c | 83 ++++++++------
drivers/gpu/drm/i915/i915_vma.h | 61 +++-------
drivers/gpu/drm/i915/intel_display.c | 42 +------
drivers/gpu/drm/i915/intel_fbdev.c | 8 +-
drivers/gpu/drm/i915/intel_guc.c | 1 +
drivers/gpu/drm/i915/intel_overlay.c | 18 +--
drivers/gpu/drm/i915/selftests/i915_vma.c | 2 +
27 files changed, 239 insertions(+), 505 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
index 22a262dfd07b..7c6fd310276e 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c
@@ -152,7 +152,6 @@ static void clear_pages_dma_fence_cb(struct dma_fence *fence,
static void clear_pages_worker(struct work_struct *work)
{
struct clear_pages_work *w = container_of(work, typeof(*w), work);
- struct drm_i915_private *i915 = w->ce->gem_context->i915;
struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
struct i915_vma *vma = w->sleeve->vma;
struct i915_request *rq;
@@ -168,11 +167,9 @@ static void clear_pages_worker(struct work_struct *work)
obj->cache_dirty = false;
}
- /* XXX: we need to kill this */
- mutex_lock(&i915->drm.struct_mutex);
err = i915_vma_pin(vma, 0, 0, PIN_USER);
if (unlikely(err))
- goto out_unlock;
+ goto out_signal;
rq = i915_request_create(w->ce);
if (IS_ERR(rq)) {
@@ -208,8 +205,6 @@ static void clear_pages_worker(struct work_struct *work)
i915_request_add(rq);
out_unpin:
i915_vma_unpin(vma);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
out_signal:
if (unlikely(err)) {
dma_fence_set_error(&w->dma, err);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 24a714853f85..da6bf27697e4 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -99,6 +99,7 @@ static void lut_close(struct i915_gem_context *ctx)
void __rcu **slot;
lockdep_assert_held(&ctx->mutex);
+ mutex_lock(&ctx->vm->mutex);
rcu_read_lock();
radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
@@ -129,13 +130,15 @@ static void lut_close(struct i915_gem_context *ctx)
radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
GEM_BUG_ON(!atomic_read(&vma->open_count));
if (atomic_dec_and_test(&vma->open_count))
- i915_vma_close(vma);
+ i915_vma_unbind(vma);
i915_vma_put(vma);
}
i915_gem_object_put(obj);
}
rcu_read_unlock();
+
+ mutex_unlock(&ctx->vm->mutex);
}
static struct intel_context *
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index a62f7f0e2947..f9044bbdd429 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -26,7 +26,7 @@ static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
{
- if (!READ_ONCE(obj->pin_global))
+ if (!atomic_read(&obj->pin_global))
return;
i915_gem_object_lock(obj);
@@ -377,16 +377,11 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
if (ret)
goto out;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- goto out;
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret == 0) {
ret = i915_gem_object_set_cache_level(obj, level);
i915_gem_object_unlock(obj);
}
- mutex_unlock(&i915->drm.struct_mutex);
out:
i915_gem_object_put(obj);
@@ -413,7 +408,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
/* Mark the global pin early so that we account for the
* display coherency whilst setting up the cache domains.
*/
- obj->pin_global++;
+ atomic_inc(&obj->pin_global);
/* The display engine is not coherent with the LLC cache on gen6. As
* a result, we make sure that the pinning that is about to occur is
@@ -463,25 +458,27 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
return vma;
err_unpin_global:
- obj->pin_global--;
+ atomic_dec(&obj->pin_global);
return vma;
}
static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
- struct i915_vma *vma;
GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
- mutex_lock(&i915->ggtt.vm.mutex);
- for_each_ggtt_vma(vma, obj) {
- if (!drm_mm_node_allocated(&vma->node))
- continue;
+ if (mutex_trylock(&i915->ggtt.vm.mutex)) {
+ struct i915_vma *vma;
- list_move_tail(&vma->vm_link, &vma->vm->bound_list);
+ for_each_ggtt_vma(vma, obj) {
+ if (!drm_mm_node_allocated(&vma->node))
+ continue;
+
+ list_move_tail(&vma->vm_link, &vma->vm->bound_list);
+ }
+ mutex_unlock(&i915->ggtt.vm.mutex);
}
- mutex_unlock(&i915->ggtt.vm.mutex);
if (i915_gem_object_is_shrinkable(obj)) {
unsigned long flags;
@@ -500,12 +497,10 @@ i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
{
struct drm_i915_gem_object *obj = vma->obj;
- assert_object_held(obj);
-
- if (WARN_ON(obj->pin_global == 0))
+ if (GEM_WARN_ON(!atomic_read(&obj->pin_global)))
return;
- if (--obj->pin_global == 0)
+ if (atomic_dec_and_test(&obj->pin_global))
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
/* Bump the LRU to try and avoid premature eviction whilst flipping */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index f12dc0f521c9..44bcb681c168 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -547,8 +547,11 @@ eb_add_vma(struct i915_execbuffer *eb,
eb_unreserve_vma(vma, vma->exec_flags);
list_add_tail(&vma->exec_link, &eb->unbound);
- if (drm_mm_node_allocated(&vma->node))
+ if (drm_mm_node_allocated(&vma->node)) {
+ mutex_lock(&vma->vm->mutex);
err = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ }
if (unlikely(err))
vma->exec_flags = NULL;
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 49cf9ad97bfc..9fa0569afb50 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -248,16 +248,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
goto err_rpm;
}
- ret = i915_mutex_lock_interruptible(dev);
- if (ret)
- goto err_reset;
-
- /* Access to snoopable pages through the GTT is incoherent. */
- if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
- ret = -EFAULT;
- goto err_unlock;
- }
-
/* Now pin it into the GTT as needed */
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE |
@@ -287,13 +277,19 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
}
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
- goto err_unlock;
+ goto err_reset;
}
- ret = i915_vma_pin_fence(vma);
+ assert_rpm_wakelock_held(i915);
+
+ ret = mutex_lock_interruptible(&vma->vm->mutex);
if (ret)
goto err_unpin;
+ ret = __i915_vma_pin_fence(vma);
+ if (ret)
+ goto err_unlock;
+
/* Finally, remap it using the new GTT offset */
ret = remap_io_mapping(area,
area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
@@ -304,7 +300,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
goto err_fence;
/* Mark as being mmapped into userspace for later revocation */
- assert_rpm_wakelock_held(i915);
if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
list_add(&obj->userfault_link, &i915->ggtt.userfault_list);
if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
@@ -316,10 +311,10 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
err_fence:
i915_vma_unpin_fence(vma);
+err_unlock:
+ mutex_unlock(&vma->vm->mutex);
err_unpin:
__i915_vma_unpin(vma);
-err_unlock:
- mutex_unlock(&dev->struct_mutex);
err_reset:
i915_reset_unlock(i915, srcu);
err_rpm:
@@ -405,7 +400,7 @@ void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
* requirement that operations to the GGTT be made holding the RPM
* wakeref.
*/
- lockdep_assert_held(&i915->drm.struct_mutex);
+ lockdep_assert_held(&i915->ggtt.vm.mutex);
wakeref = intel_runtime_pm_get(i915);
if (!obj->userfault_count)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 7ea8013d108f..0fc54924b33a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -403,7 +403,8 @@ static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
return true;
- return obj->pin_global; /* currently in use by HW, keep flushed */
+ /* Currently in use by HW? Keep flushed. */
+ return atomic_read(&obj->pin_global);
}
static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 8f61d7a93078..f792953b8a71 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -159,7 +159,7 @@ struct drm_i915_gem_object {
/** Count of VMA actually bound by this object */
atomic_t bind_count;
/** Count of how many global VMA are currently pinned for use by HW */
- unsigned int pin_global;
+ atomic_t pin_global;
struct {
struct mutex lock; /* protects the pages and their use */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
index 7445e6f662ac..f833e1bd7c2f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
@@ -16,40 +16,6 @@
#include "i915_trace.h"
-static bool shrinker_lock(struct drm_i915_private *i915,
- unsigned int flags,
- bool *unlock)
-{
- struct mutex *m = &i915->drm.struct_mutex;
-
- switch (mutex_trylock_recursive(m)) {
- case MUTEX_TRYLOCK_RECURSIVE:
- *unlock = false;
- return true;
-
- case MUTEX_TRYLOCK_FAILED:
- *unlock = false;
- if (flags & I915_SHRINK_ACTIVE &&
- mutex_lock_killable_nested(m, I915_MM_SHRINKER) == 0)
- *unlock = true;
- return *unlock;
-
- case MUTEX_TRYLOCK_SUCCESS:
- *unlock = true;
- return true;
- }
-
- BUG();
-}
-
-static void shrinker_unlock(struct drm_i915_private *i915, bool unlock)
-{
- if (!unlock)
- return;
-
- mutex_unlock(&i915->drm.struct_mutex);
-}
-
static bool swap_available(void)
{
return get_nr_swap_pages() > 0;
@@ -78,7 +44,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
* To simplify the scan, and to avoid walking the list of vma under the
* object, we just check the count of its permanently pinned.
*/
- if (READ_ONCE(obj->pin_global))
+ if (atomic_read(&obj->pin_global))
return false;
/* We can only return physical pages to the system if we can either
@@ -154,10 +120,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
intel_wakeref_t wakeref = 0;
unsigned long count = 0;
unsigned long scanned = 0;
- bool unlock;
-
- if (!shrinker_lock(i915, shrink, &unlock))
- return 0;
/*
* When shrinking the active list, also consider active contexts.
@@ -174,7 +136,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
MAX_SCHEDULE_TIMEOUT);
trace_i915_gem_shrink(i915, target, shrink);
- i915_retire_requests(i915);
/*
* Unbinding of objects will require HW access; Let us not wake the
@@ -216,13 +177,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
INIT_LIST_HEAD(&still_in_list);
- /*
- * We serialize our access to unreferenced objects through
- * the use of the struct_mutex. While the objects are not
- * yet freed (due to RCU then a workqueue) we still want
- * to be able to shrink their pages, so they remain on
- * the unbound/bound list until actually freed.
- */
spin_lock_irqsave(&i915->mm.obj_lock, flags);
while (count < target &&
(obj = list_first_entry_or_null(phase->list,
@@ -270,10 +224,6 @@ i915_gem_shrink(struct drm_i915_private *i915,
if (shrink & I915_SHRINK_BOUND)
intel_runtime_pm_put(i915, wakeref);
- i915_retire_requests(i915);
-
- shrinker_unlock(i915, unlock);
-
if (nr_scanned)
*nr_scanned += scanned;
return count;
@@ -343,13 +293,9 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
struct drm_i915_private *i915 =
container_of(shrinker, struct drm_i915_private, mm.shrinker);
unsigned long freed;
- bool unlock;
sc->nr_scanned = 0;
- if (!shrinker_lock(i915, 0, &unlock))
- return SHRINK_STOP;
-
freed = i915_gem_shrink(i915,
sc->nr_to_scan,
&sc->nr_scanned,
@@ -370,8 +316,6 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
}
}
- shrinker_unlock(i915, unlock);
-
return sc->nr_scanned ? freed : SHRINK_STOP;
}
@@ -423,16 +367,12 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
struct i915_vma *vma, *next;
unsigned long freed_pages = 0;
intel_wakeref_t wakeref;
- bool unlock;
-
- if (!shrinker_lock(i915, 0, &unlock))
- return NOTIFY_DONE;
/* Force everything onto the inactive lists */
if (i915_gem_wait_for_idle(i915,
I915_WAIT_LOCKED,
MAX_SCHEDULE_TIMEOUT))
- goto out;
+ return NOTIFY_DONE;
with_intel_runtime_pm(i915, wakeref)
freed_pages += i915_gem_shrink(i915, -1UL, NULL,
@@ -449,16 +389,11 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
if (!vma->iomap || i915_vma_is_active(vma))
continue;
- mutex_unlock(&i915->ggtt.vm.mutex);
if (i915_vma_unbind(vma) == 0)
freed_pages += count;
- mutex_lock(&i915->ggtt.vm.mutex);
}
mutex_unlock(&i915->ggtt.vm.mutex);
-out:
- shrinker_unlock(i915, unlock);
-
*(unsigned long *)ptr += freed_pages;
return NOTIFY_DONE;
}
@@ -500,37 +435,13 @@ void i915_gem_shrinker_unregister(struct drm_i915_private *i915)
void i915_gem_shrinker_taints_mutex(struct drm_i915_private *i915,
struct mutex *mutex)
{
- bool unlock = false;
-
if (!IS_ENABLED(CONFIG_LOCKDEP))
return;
- if (!lockdep_is_held_type(&i915->drm.struct_mutex, -1)) {
- mutex_acquire(&i915->drm.struct_mutex.dep_map,
- I915_MM_NORMAL, 0, _RET_IP_);
- unlock = true;
- }
-
fs_reclaim_acquire(GFP_KERNEL);
- /*
- * As we invariably rely on the struct_mutex within the shrinker,
- * but have a complicated recursion dance, taint all the mutexes used
- * within the shrinker with the struct_mutex. For completeness, we
- * taint with all subclass of struct_mutex, even though we should
- * only need tainting by I915_MM_NORMAL to catch possible ABBA
- * deadlocks from using struct_mutex inside @mutex.
- */
- mutex_acquire(&i915->drm.struct_mutex.dep_map,
- I915_MM_SHRINKER, 0, _RET_IP_);
-
mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_);
mutex_release(&mutex->dep_map, 0, _RET_IP_);
- mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_);
-
fs_reclaim_release(GFP_KERNEL);
-
- if (unlock)
- mutex_release(&i915->drm.struct_mutex.dep_map, 0, _RET_IP_);
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
index ca0c2f451742..f919dbb36edb 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
@@ -212,15 +212,18 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
- lockdep_assert_held(&i915->drm.struct_mutex);
if ((tiling | stride) == obj->tiling_and_stride)
return 0;
- if (i915_gem_object_is_framebuffer(obj))
+ i915_gem_object_lock(obj);
+ if (i915_gem_object_is_framebuffer(obj)) {
+ i915_gem_object_unlock(obj);
return -EBUSY;
+ }
- /* We need to rebind the object if its current allocation
+ /*
+ * We need to rebind the object if its current allocation
* no longer meets the alignment restrictions for its new
* tiling mode. Otherwise we can just leave it alone, but
* need to ensure that any fence register is updated before
@@ -233,17 +236,37 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
* whilst executing a fenced command for an untiled object.
*/
- err = i915_gem_object_fence_prepare(obj, tiling, stride);
- if (err)
+ err = mutex_lock_interruptible(&i915->ggtt.vm.mutex);
+ if (err) {
+ i915_gem_object_unlock(obj);
return err;
+ }
- i915_gem_object_lock(obj);
- if (i915_gem_object_is_framebuffer(obj)) {
+ err = i915_gem_object_fence_prepare(obj, tiling, stride);
+ if (err) {
+ mutex_unlock(&i915->ggtt.vm.mutex);
i915_gem_object_unlock(obj);
- return -EBUSY;
+ return err;
}
- /* If the memory has unknown (i.e. varying) swizzling, we pin the
+ for_each_ggtt_vma(vma, obj) {
+ vma->fence_size =
+ i915_gem_fence_size(i915, vma->size, tiling, stride);
+ vma->fence_alignment =
+ i915_gem_fence_alignment(i915,
+ vma->size, tiling, stride);
+
+ if (vma->fence)
+ vma->fence->dirty = true;
+ }
+
+ /* Force the fence to be reacquired for GTT access */
+ i915_gem_object_release_mmap(obj);
+
+ mutex_unlock(&i915->ggtt.vm.mutex);
+
+ /*
+ * If the memory has unknown (i.e. varying) swizzling, we pin the
* pages to prevent them being swapped out and causing corruption
* due to the change in swizzling.
*/
@@ -264,23 +287,9 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
}
mutex_unlock(&obj->mm.lock);
- for_each_ggtt_vma(vma, obj) {
- vma->fence_size =
- i915_gem_fence_size(i915, vma->size, tiling, stride);
- vma->fence_alignment =
- i915_gem_fence_alignment(i915,
- vma->size, tiling, stride);
-
- if (vma->fence)
- vma->fence->dirty = true;
- }
-
obj->tiling_and_stride = tiling | stride;
i915_gem_object_unlock(obj);
- /* Force the fence to be reacquired for GTT access */
- i915_gem_object_release_mmap(obj);
-
/* Try to preallocate memory required to save swizzling on put-pages */
if (i915_gem_object_needs_bit17_swizzle(obj)) {
if (!obj->bit_17) {
@@ -364,12 +373,7 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
}
}
- err = mutex_lock_interruptible(&dev->struct_mutex);
- if (err)
- goto err;
-
err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
- mutex_unlock(&dev->struct_mutex);
/* We have to maintain this existing ABI... */
args->stride = i915_gem_object_get_stride(obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
index 528b61678334..f093deaeb5c0 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
@@ -93,7 +93,6 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
struct i915_mmu_notifier *mn =
container_of(_mn, struct i915_mmu_notifier, mn);
struct interval_tree_node *it;
- struct mutex *unlock = NULL;
unsigned long end;
int ret = 0;
@@ -130,32 +129,12 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
}
spin_unlock(&mn->lock);
- if (!unlock) {
- unlock = &mn->mm->i915->drm.struct_mutex;
-
- switch (mutex_trylock_recursive(unlock)) {
- default:
- case MUTEX_TRYLOCK_FAILED:
- if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) {
- i915_gem_object_put(obj);
- return -EINTR;
- }
- /* fall through */
- case MUTEX_TRYLOCK_SUCCESS:
- break;
-
- case MUTEX_TRYLOCK_RECURSIVE:
- unlock = ERR_PTR(-EEXIST);
- break;
- }
- }
-
ret = i915_gem_object_unbind(obj);
if (ret == 0)
ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
i915_gem_object_put(obj);
if (ret)
- goto unlock;
+ return ret;
spin_lock(&mn->lock);
@@ -168,12 +147,7 @@ userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
}
spin_unlock(&mn->lock);
-unlock:
- if (!IS_ERR_OR_NULL(unlock))
- mutex_unlock(unlock);
-
return ret;
-
}
static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index a109904faa57..967256d7e8a6 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -110,7 +110,7 @@ static int __context_pin_state(struct i915_vma *vma)
* And mark it as a globally pinned object to let the shrinker know
* it cannot reclaim the object until we release it.
*/
- vma->obj->pin_global++;
+ atomic_inc(&vma->obj->pin_global);
vma->obj->mm.dirty = true;
return 0;
@@ -118,7 +118,8 @@ static int __context_pin_state(struct i915_vma *vma)
static void __context_unpin_state(struct i915_vma *vma)
{
- vma->obj->pin_global--;
+ GEM_BUG_ON(!atomic_read(&vma->obj->pin_global));
+ atomic_dec(&vma->obj->pin_global);
__i915_vma_unpin(vma);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
index d3783beaf132..95d9adc7e2e9 100644
--- a/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/gt/intel_ringbuffer.c
@@ -1183,7 +1183,7 @@ int intel_ring_pin(struct intel_ring *ring)
goto unpin_ring;
}
- vma->obj->pin_global++;
+ atomic_read(&vma->obj->pin_global);
ring->vaddr = addr;
return 0;
@@ -1219,7 +1219,7 @@ void intel_ring_unpin(struct intel_ring *ring)
i915_gem_object_unpin_map(ring->vma->obj);
ring->vaddr = NULL;
- ring->vma->obj->pin_global--;
+ atomic_dec(&ring->vma->obj->pin_global);
i915_vma_unpin(ring->vma);
i915_timeline_unpin(ring->timeline);
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
index 2bd9dcebd48c..a2d7a1df820f 100644
--- a/drivers/gpu/drm/i915/gvt/aperture_gm.c
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -61,14 +61,14 @@ static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm)
flags = PIN_MAPPABLE;
}
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
mmio_hw_access_pre(dev_priv);
ret = i915_gem_gtt_insert(&dev_priv->ggtt.vm, node,
size, I915_GTT_PAGE_SIZE,
I915_COLOR_UNEVICTABLE,
start, end, flags);
mmio_hw_access_post(dev_priv);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
if (ret)
gvt_err("fail to alloc %s gm space from host\n",
high_gm ? "high" : "low");
@@ -98,9 +98,9 @@ static int alloc_vgpu_gm(struct intel_vgpu *vgpu)
return 0;
out_free_aperture:
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
drm_mm_remove_node(&vgpu->gm.low_gm_node);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
return ret;
}
@@ -108,10 +108,10 @@ static void free_vgpu_gm(struct intel_vgpu *vgpu)
{
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
- mutex_lock(&dev_priv->drm.struct_mutex);
+ mutex_lock(&dev_priv->ggtt.vm.mutex);
drm_mm_remove_node(&vgpu->gm.low_gm_node);
drm_mm_remove_node(&vgpu->gm.high_gm_node);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ mutex_unlock(&dev_priv->ggtt.vm.mutex);
}
/**
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index b54f7b79a53c..cbeaa53117af 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -76,7 +76,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
static char get_pin_flag(struct drm_i915_gem_object *obj)
{
- return obj->pin_global ? 'p' : ' ';
+ return atomic_read(&obj->pin_global) ? 'p' : ' ';
}
static char get_tiling_flag(struct drm_i915_gem_object *obj)
@@ -218,7 +218,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
seq_printf(m, " (pinned x %d)", pin_count);
if (obj->stolen)
seq_printf(m, " (stolen: %08llx)", obj->stolen->start);
- if (obj->pin_global)
+ if (atomic_read(&obj->pin_global))
seq_printf(m, " (global)");
engine = i915_gem_object_last_write_engine(obj);
@@ -1604,11 +1604,6 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
struct drm_device *dev = &dev_priv->drm;
struct intel_framebuffer *fbdev_fb = NULL;
struct drm_framebuffer *drm_fb;
- int ret;
-
- ret = mutex_lock_interruptible(&dev->struct_mutex);
- if (ret)
- return ret;
#ifdef CONFIG_DRM_FBDEV_EMULATION
if (dev_priv->fbdev && dev_priv->fbdev->helper.fb) {
@@ -1643,7 +1638,6 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
seq_putc(m, '\n');
}
mutex_unlock(&dev->mode_config.fb_lock);
- mutex_unlock(&dev->struct_mutex);
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index fd448afae492..9e835355eb93 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -60,20 +60,31 @@
#include "intel_pm.h"
static int
-insert_mappable_node(struct i915_ggtt *ggtt,
- struct drm_mm_node *node, u32 size)
+insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
{
+ int ret;
+
memset(node, 0, sizeof(*node));
- return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
- size, 0, I915_COLOR_UNEVICTABLE,
- 0, ggtt->mappable_end,
- DRM_MM_INSERT_LOW);
+
+ ret = mutex_lock_interruptible(&ggtt->vm.mutex);
+ if (ret)
+ return ret;
+
+ ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
+ size, 0, I915_COLOR_UNEVICTABLE,
+ 0, ggtt->mappable_end,
+ DRM_MM_INSERT_LOW);
+ mutex_unlock(&ggtt->vm.mutex);
+
+ return ret;
}
static void
-remove_mappable_node(struct drm_mm_node *node)
+remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
{
+ mutex_lock(&ggtt->vm.mutex);
drm_mm_remove_node(node);
+ mutex_unlock(&ggtt->vm.mutex);
}
int
@@ -84,8 +95,11 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
struct drm_i915_gem_get_aperture *args = data;
struct i915_vma *vma;
u64 pinned;
+ int ret;
- mutex_lock(&ggtt->vm.mutex);
+ ret = mutex_lock_interruptible(&ggtt->vm.mutex);
+ if (ret)
+ return ret;
pinned = ggtt->vm.reserved;
list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
@@ -106,8 +120,6 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
LIST_HEAD(still_in_list);
int ret = 0;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
spin_lock(&obj->vma.lock);
while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
struct i915_vma,
@@ -117,7 +129,11 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
list_move_tail(&vma->obj_link, &still_in_list);
spin_unlock(&obj->vma.lock);
- ret = i915_vma_unbind(vma);
+ ret = mutex_lock_interruptible(&vma->vm->mutex);
+ if (!ret) {
+ ret = i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ }
i915_vma_put(vma);
spin_lock(&obj->vma.lock);
@@ -375,10 +391,6 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
u64 remain, offset;
int ret;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
wakeref = intel_runtime_pm_get(i915);
vma = ERR_PTR(-ENODEV);
if (!i915_gem_object_is_tiled(obj)) {
@@ -393,12 +405,10 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
} else {
ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
if (ret)
- goto out_unlock;
+ goto out_rpm;
GEM_BUG_ON(!node.allocated);
}
- mutex_unlock(&i915->drm.struct_mutex);
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
@@ -454,17 +464,15 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
i915_gem_object_unlock_fence(obj, fence);
out_unpin:
- mutex_lock(&i915->drm.struct_mutex);
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
-out_unlock:
+out_rpm:
intel_runtime_pm_put(i915, wakeref);
- mutex_unlock(&i915->drm.struct_mutex);
return ret;
}
@@ -571,10 +579,6 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
void __user *user_data;
int ret;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
if (i915_gem_object_has_struct_page(obj)) {
/*
* Avoid waking the device up if we can fallback, as
@@ -584,10 +588,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
* using the cache bypass of indirect GGTT access.
*/
wakeref = intel_runtime_pm_get_if_in_use(i915);
- if (!wakeref) {
- ret = -EFAULT;
- goto out_unlock;
- }
+ if (!wakeref)
+ return -EFAULT;
} else {
/* No backing pages, no fallback, we must force GGTT access */
wakeref = intel_runtime_pm_get(i915);
@@ -610,8 +612,6 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
GEM_BUG_ON(!node.allocated);
}
- mutex_unlock(&i915->drm.struct_mutex);
-
ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
@@ -674,18 +674,15 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
i915_gem_object_unlock_fence(obj, fence);
out_unpin:
- mutex_lock(&i915->drm.struct_mutex);
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
- remove_mappable_node(&node);
+ remove_mappable_node(ggtt, &node);
} else {
i915_vma_unpin(vma);
}
out_rpm:
intel_runtime_pm_put(i915, wakeref);
-out_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
return ret;
}
@@ -1036,8 +1033,6 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
if (flags & PIN_MAPPABLE &&
(!view || view->type == I915_GGTT_VIEW_NORMAL)) {
/* If the required space is larger than the available
@@ -1074,14 +1069,28 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
if (IS_ERR(vma))
return vma;
+ ret = i915_gem_object_pin_pages(obj);
+ if (ret)
+ return ERR_PTR(ret);
+
+ ret = mutex_lock_interruptible(&vm->mutex);
+ if (ret) {
+ vma = ERR_PTR(ret);
+ goto unpin;
+ }
+
if (i915_vma_misplaced(vma, size, alignment, flags)) {
if (flags & PIN_NONBLOCK) {
- if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
- return ERR_PTR(-ENOSPC);
+ if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
+ vma = ERR_PTR(-ENOSPC);
+ goto unlock;
+ }
if (flags & PIN_MAPPABLE &&
- vma->fence_size > dev_priv->ggtt.mappable_end / 2)
- return ERR_PTR(-ENOSPC);
+ vma->fence_size > dev_priv->ggtt.mappable_end / 2) {
+ vma = ERR_PTR(-ENOSPC);
+ goto unlock;
+ }
}
WARN(i915_vma_is_pinned(vma),
@@ -1094,15 +1103,20 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
ret = i915_vma_unbind(vma);
if (ret) {
i915_vma_put(vma);
- return ERR_PTR(ret);
+ vma = ERR_PTR(ret);
+ goto unlock;
}
}
- ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
+ ret = __i915_vma_do_pin(vma, size, alignment, flags | PIN_GLOBAL);
i915_vma_put(vma);
if (ret)
- return ERR_PTR(ret);
+ vma = ERR_PTR(ret);
+unlock:
+ mutex_unlock(&vm->mutex);
+unpin:
+ i915_gem_object_unpin_pages(obj);
return vma;
}
@@ -1401,7 +1415,9 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
* from the GTT to prevent such accidents and reclaim the
* space.
*/
+ mutex_lock(&state->vm->mutex);
err = i915_vma_unbind(state);
+ mutex_unlock(&state->vm->mutex);
if (err)
goto err_active;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index a5783c4cb98b..0202f58d8bb5 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -48,8 +48,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
* bound by their active reference.
*/
return i915_gem_wait_for_idle(i915,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
+ I915_WAIT_INTERRUPTIBLE,
MAX_SCHEDULE_TIMEOUT);
}
@@ -108,7 +107,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
struct i915_vma *active;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
@@ -131,15 +130,6 @@ i915_gem_evict_something(struct i915_address_space *vm,
min_size, alignment, cache_level,
start, end, mode);
- /*
- * Retire before we search the active list. Although we have
- * reasonable accuracy in our retirement lists, we may have
- * a stray pin (preventing eviction) that can only be resolved by
- * retiring.
- */
- if (!(flags & PIN_NONBLOCK))
- i915_retire_requests(dev_priv);
-
search_again:
active = NULL;
INIT_LIST_HEAD(&eviction_list);
@@ -273,20 +263,12 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
bool check_color;
int ret = 0;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
trace_i915_gem_evict_node(vm, target, flags);
- /* Retire before we search the active list. Although we have
- * reasonable accuracy in our retirement lists, we may have
- * a stray pin (preventing eviction) that can only be resolved by
- * retiring.
- */
- if (!(flags & PIN_NONBLOCK))
- i915_retire_requests(vm->i915);
-
check_color = vm->mm.color_adjust;
if (check_color) {
/* Expand search to cover neighbouring guard pages (or lack!) */
@@ -384,7 +366,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
struct i915_vma *vma, *next;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict_vm(vm);
/* Switch back to the default context in order to unpin
@@ -399,7 +381,6 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
}
INIT_LIST_HEAD(&eviction_list);
- mutex_lock(&vm->mutex);
list_for_each_entry(vma, &vm->bound_list, vm_link) {
if (i915_vma_is_pinned(vma))
continue;
@@ -407,7 +388,6 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
__i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
- mutex_unlock(&vm->mutex);
ret = 0;
list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
index e596dd5ae1e2..17215de2a8b8 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence_reg.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
@@ -331,7 +331,7 @@ static struct i915_fence_reg *fence_find(struct drm_i915_private *i915)
return ERR_PTR(-EDEADLK);
}
-static int __i915_vma_pin_fence(struct i915_vma *vma)
+int __i915_vma_pin_fence(struct i915_vma *vma)
{
struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
struct i915_fence_reg *fence;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index fb484685ff1b..c7d7959f761f 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1929,59 +1929,11 @@ static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
free_pt(&ppgtt->base.vm, pt);
}
-struct gen6_ppgtt_cleanup_work {
- struct work_struct base;
- struct i915_vma *vma;
-};
-
-static void gen6_ppgtt_cleanup_work(struct work_struct *wrk)
-{
- struct gen6_ppgtt_cleanup_work *work =
- container_of(wrk, typeof(*work), base);
-
- /* Side note, vma->vm is the GGTT not the ppgtt we just destroyed! */
- i915_vma_destroy(work->vma);
-
- kfree(work);
-}
-
-static int nop_set_pages(struct i915_vma *vma)
-{
- return -ENODEV;
-}
-
-static void nop_clear_pages(struct i915_vma *vma)
-{
-}
-
-static int nop_bind(struct i915_vma *vma,
- enum i915_cache_level cache_level,
- u32 unused)
-{
- return -ENODEV;
-}
-
-static void nop_unbind(struct i915_vma *vma)
-{
-}
-
-static const struct i915_vma_ops nop_vma_ops = {
- .set_pages = nop_set_pages,
- .clear_pages = nop_clear_pages,
- .bind_vma = nop_bind,
- .unbind_vma = nop_unbind,
-};
-
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
- struct gen6_ppgtt_cleanup_work *work = ppgtt->work;
- /* FIXME remove the struct_mutex to bring the locking under control */
- INIT_WORK(&work->base, gen6_ppgtt_cleanup_work);
- work->vma = ppgtt->vma;
- work->vma->ops = &nop_vma_ops;
- schedule_work(&work->base);
+ i915_vma_destroy(ppgtt->vma);
gen6_ppgtt_free_pd(ppgtt);
gen6_ppgtt_free_scratch(vm);
@@ -2157,15 +2109,9 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
- ppgtt->work = kmalloc(sizeof(*ppgtt->work), GFP_KERNEL);
- if (!ppgtt->work) {
- err = -ENOMEM;
- goto err_free;
- }
-
err = gen6_ppgtt_init_scratch(ppgtt);
if (err)
- goto err_work;
+ goto err_free;
ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
if (IS_ERR(ppgtt->vma)) {
@@ -2177,8 +2123,6 @@ static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
err_scratch:
gen6_ppgtt_free_scratch(&ppgtt->base.vm);
-err_work:
- kfree(ppgtt->work);
err_free:
kfree(ppgtt);
return ERR_PTR(err);
@@ -2895,11 +2839,12 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
ggtt->vm.closed = true;
- mutex_lock(&dev_priv->drm.struct_mutex);
fini_aliasing_ppgtt(dev_priv);
+ mutex_lock(&ggtt->vm.mutex);
list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
WARN_ON(i915_vma_unbind(vma));
+ mutex_unlock(&ggtt->vm.mutex);
if (drm_mm_node_allocated(&ggtt->error_capture))
drm_mm_remove_node(&ggtt->error_capture);
@@ -2919,8 +2864,6 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
__pagevec_release(pvec);
}
- mutex_unlock(&dev_priv->drm.struct_mutex);
-
arch_phys_wc_del(ggtt->mtrr);
io_mapping_fini(&ggtt->iomap);
@@ -3523,7 +3466,6 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
* beyond the end of the batch buffer, across the page boundary,
* and beyond the end of the GTT if we do not provide a guard.
*/
- mutex_lock(&dev_priv->drm.struct_mutex);
i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
ggtt->vm.is_ggtt = true;
@@ -3533,7 +3475,6 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
if (!HAS_LLC(dev_priv) && !HAS_PPGTT(dev_priv))
ggtt->vm.mm.color_adjust = i915_gtt_color_adjust;
- mutex_unlock(&dev_priv->drm.struct_mutex);
if (!io_mapping_init_wc(&dev_priv->ggtt.iomap,
dev_priv->ggtt.gmadr.start,
@@ -3612,10 +3553,8 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
if (!(vma->flags & I915_VMA_GLOBAL_BIND))
continue;
- mutex_unlock(&ggtt->vm.mutex);
-
if (!i915_vma_unbind(vma))
- goto lock;
+ continue;
WARN_ON(i915_vma_bind(vma,
obj ? obj->cache_level : 0,
@@ -3625,9 +3564,6 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
i915_gem_object_unlock(obj);
}
-
-lock:
- mutex_lock(&ggtt->vm.mutex);
}
ggtt->vm.closed = false;
@@ -4024,7 +3960,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
u64 offset;
int err;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!size);
GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(alignment && !is_power_of_2(alignment));
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 889c4e88c598..d957f1a5b6a3 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -438,8 +438,6 @@ struct gen6_ppgtt {
unsigned int pin_count;
bool scan_for_unused_pt;
-
- struct gen6_ppgtt_cleanup_work *work;
};
#define __to_gen6_ppgtt(base) container_of(base, struct gen6_ppgtt, base)
@@ -692,7 +690,6 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
#define PIN_OFFSET_BIAS BIT_ULL(6)
#define PIN_OFFSET_FIXED BIT_ULL(7)
-#define PIN_MBZ BIT_ULL(8) /* I915_VMA_PIN_OVERFLOW */
#define PIN_GLOBAL BIT_ULL(9) /* I915_VMA_GLOBAL_BIND */
#define PIN_USER BIT_ULL(10) /* I915_VMA_LOCAL_BIND */
#define PIN_UPDATE BIT_ULL(11)
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index d92ddfada262..891b60f49843 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1347,13 +1347,9 @@ static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
static void
free_oa_buffer(struct drm_i915_private *i915)
{
- mutex_lock(&i915->drm.struct_mutex);
-
i915_vma_unpin_and_release(&i915->perf.oa.oa_buffer.vma,
I915_VMA_RELEASE_MAP);
- mutex_unlock(&i915->drm.struct_mutex);
-
i915->perf.oa.oa_buffer.vaddr = NULL;
}
@@ -1505,19 +1501,12 @@ static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma))
return -ENODEV;
- ret = i915_mutex_lock_interruptible(&dev_priv->drm);
- if (ret)
- return ret;
-
BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE);
BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M);
bo = i915_gem_object_create_shmem(dev_priv, OA_BUFFER_SIZE);
- if (IS_ERR(bo)) {
- DRM_ERROR("Failed to allocate OA buffer\n");
- ret = PTR_ERR(bo);
- goto unlock;
- }
+ if (IS_ERR(bo))
+ return PTR_ERR(bo);
i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC);
@@ -1539,8 +1528,7 @@ static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n",
i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma),
dev_priv->perf.oa.oa_buffer.vaddr);
-
- goto unlock;
+ return 0;
err_unpin:
__i915_vma_unpin(vma);
@@ -1551,8 +1539,6 @@ static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
dev_priv->perf.oa.oa_buffer.vaddr = NULL;
dev_priv->perf.oa.oa_buffer.vma = NULL;
-unlock:
- mutex_unlock(&dev_priv->drm.struct_mutex);
return ret;
}
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index b5875db6f2c0..b06e4f5db6a8 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -318,8 +318,6 @@ vma_lookup(struct drm_i915_gem_object *obj,
* Once created, the VMA is kept until either the object is freed, or the
* address space is closed.
*
- * Must be called with struct_mutex held.
- *
* Returns the vma, or an error pointer.
*/
struct i915_vma *
@@ -458,7 +456,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
/* Access through the GTT requires the device to be awake. */
assert_rpm_wakelock_held(vma->vm->i915);
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ mutex_lock(&vma->vm->mutex);
+
if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
err = -ENODEV;
goto err;
@@ -482,16 +481,19 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
__i915_vma_pin(vma);
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma);
if (err)
goto err_unpin;
i915_vma_set_ggtt_write(vma);
+ mutex_unlock(&vma->vm->mutex);
+
return ptr;
err_unpin:
__i915_vma_unpin(vma);
err:
+ mutex_unlock(&vma->vm->mutex);
return IO_ERR_PTR(err);
}
@@ -507,8 +509,6 @@ void i915_vma_flush_writes(struct i915_vma *vma)
void i915_vma_unpin_iomap(struct i915_vma *vma)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
-
GEM_BUG_ON(vma->iomap == NULL);
i915_vma_flush_writes(vma);
@@ -684,10 +684,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
}
if (vma->obj) {
- ret = i915_gem_object_pin_pages_async(vma->obj);
- if (ret)
- return ret;
-
+ __i915_gem_object_pin_pages(vma->obj);
cache_level = vma->obj->cache_level;
} else {
cache_level = 0;
@@ -752,9 +749,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
- mutex_lock(&vma->vm->mutex);
list_add_tail(&vma->vm_link, &vma->vm->bound_list);
- mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
atomic_inc(&vma->obj->bind_count);
@@ -777,10 +772,8 @@ i915_vma_remove(struct i915_vma *vma)
vma->ops->clear_pages(vma);
- mutex_lock(&vma->vm->mutex);
drm_mm_remove_node(&vma->node);
list_del(&vma->vm_link);
- mutex_unlock(&vma->vm->mutex);
/*
* Since the unbound list is global, only move to that list if
@@ -801,25 +794,20 @@ i915_vma_remove(struct i915_vma *vma)
}
}
-int __i915_vma_do_pin(struct i915_vma *vma,
- u64 size, u64 alignment, u64 flags)
+int __i915_vma_do_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
const unsigned int bound = vma->flags;
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
- GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
- GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
+ lockdep_assert_held(&vma->vm->mutex);
- if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
- ret = -EBUSY;
- goto err_unpin;
- }
+ if (atomic_read(&vma->pin_count))
+ goto out;
if ((bound & I915_VMA_BIND_MASK) == 0) {
ret = i915_vma_insert(vma, size, alignment, flags);
if (ret)
- goto err_unpin;
+ return ret;
}
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
@@ -833,6 +821,9 @@ int __i915_vma_do_pin(struct i915_vma *vma,
__i915_vma_set_map_and_fenceable(vma);
GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
+
+out:
+ atomic_inc(&vma->pin_count);
return 0;
err_remove:
@@ -841,11 +832,34 @@ int __i915_vma_do_pin(struct i915_vma *vma,
GEM_BUG_ON(vma->pages);
GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK);
}
-err_unpin:
- __i915_vma_unpin(vma);
return ret;
}
+int i915_vma_do_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+{
+ int err;
+
+ GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
+ GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
+
+ if (vma->obj) {
+ err = i915_gem_object_pin_pages_async(vma->obj);
+ if (err)
+ return err;
+ }
+
+ err = mutex_lock_interruptible(&vma->vm->mutex);
+ if (!err) {
+ err = __i915_vma_do_pin(vma, size, alignment, flags);
+ mutex_unlock(&vma->vm->mutex);
+ }
+
+ if (vma->obj)
+ i915_gem_object_unpin_pages(vma->obj);
+
+ return err;
+}
+
void i915_vma_close(struct i915_vma *vma)
{
struct drm_i915_private *i915 = vma->vm->i915;
@@ -886,10 +900,10 @@ void i915_vma_reopen(struct i915_vma *vma)
void i915_vma_destroy(struct i915_vma *vma)
{
- mutex_lock(&vma->vm->i915->drm.struct_mutex);
- vma->flags &= ~I915_VMA_PIN_MASK;
+ mutex_lock(&vma->vm->mutex);
+ atomic_set(&vma->pin_count, 0);
WARN_ON(i915_vma_unbind(vma));
- mutex_unlock(&vma->vm->i915->drm.struct_mutex);
+ mutex_unlock(&vma->vm->mutex);
i915_vma_put(vma);
}
@@ -925,11 +939,15 @@ void i915_vma_parked(struct drm_i915_private *i915)
spin_lock_irq(&i915->gt.closed_lock);
list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) {
+ i915_vma_get(vma);
list_del_init(&vma->closed_link);
spin_unlock_irq(&i915->gt.closed_lock);
+ mutex_lock(&vma->vm->mutex);
i915_vma_unbind(vma);
+ mutex_unlock(&vma->vm->mutex);
+ i915_vma_put(vma);
spin_lock_irq(&i915->gt.closed_lock);
}
spin_unlock_irq(&i915->gt.closed_lock);
@@ -951,7 +969,8 @@ void i915_vma_revoke_mmap(struct i915_vma *vma)
struct drm_vma_offset_node *node = &vma->obj->base.vma_node;
u64 vma_offset;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
+ GEM_BUG_ON(!i915_vma_is_ggtt(vma));
if (!i915_vma_has_userfault(vma))
return;
@@ -1034,7 +1053,7 @@ int i915_vma_unbind(struct i915_vma *vma)
{
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
if (vma->async.dma &&
dma_fence_wait_timeout(vma->async.dma, true,
@@ -1065,9 +1084,7 @@ int i915_vma_unbind(struct i915_vma *vma)
GEM_BUG_ON(i915_vma_has_ggtt_write(vma));
/* release the fence reg _after_ flushing */
- mutex_lock(&vma->vm->mutex);
ret = i915_vma_revoke_fence(vma);
- mutex_unlock(&vma->vm->mutex);
if (ret)
return ret;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index f9f4cbb99e62..140d12f67cf8 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -71,39 +71,15 @@ struct i915_vma {
u32 fence_size;
u32 fence_alignment;
+ atomic_t pin_count;
atomic_t open_count;
unsigned long flags;
- /**
- * How many users have pinned this object in GTT space.
- *
- * This is a tightly bound, fairly small number of users, so we
- * stuff inside the flags field so that we can both check for overflow
- * and detect a no-op i915_vma_pin() in a single check, while also
- * pinning the vma.
- *
- * The worst case display setup would have the same vma pinned for
- * use on each plane on each crtc, while also building the next atomic
- * state and holding a pin for the length of the cleanup queue. In the
- * future, the flip queue may be increased from 1.
- * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84
- *
- * For GEM, the number of concurrent users for pwrite/pread is
- * unbounded. For execbuffer, it is currently one but will in future
- * be extended to allow multiple clients to pin vma concurrently.
- *
- * We also use suballocated pages, with each suballocation claiming
- * its own pin on the shared vma. At present, this is limited to
- * exclusive cachelines of a single page, so a maximum of 64 possible
- * users.
- */
-#define I915_VMA_PIN_MASK 0xff
-#define I915_VMA_PIN_OVERFLOW BIT(8)
/** Flags and address space this VMA is bound to */
#define I915_VMA_GLOBAL_BIND BIT(9)
#define I915_VMA_LOCAL_BIND BIT(10)
-#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
-#define I915_VMA_ALLOC_BIND I915_VMA_PIN_OVERFLOW /* not stored */
+#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)
+#define I915_VMA_ALLOC_BIND BIT(0) /* not stored */
#define I915_VMA_GGTT BIT(11)
#define I915_VMA_CAN_FENCE BIT(12)
@@ -328,30 +304,27 @@ static inline void i915_vma_unlock(struct i915_vma *vma)
reservation_object_unlock(vma->resv);
}
-int __i915_vma_do_pin(struct i915_vma *vma,
- u64 size, u64 alignment, u64 flags);
+int __i915_vma_do_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
+int i915_vma_do_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
+
static inline int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
- /* Pin early to prevent the shrinker/eviction logic from destroying
- * our vma as we insert and bind.
- */
- if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) {
+ if (atomic_add_unless(&vma->pin_count, 1, 0)) {
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
return 0;
}
- return __i915_vma_do_pin(vma, size, alignment, flags);
+ return i915_vma_do_pin(vma, size, alignment, flags);
}
static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
- return vma->flags & I915_VMA_PIN_MASK;
+ return atomic_read(&vma->pin_count);
}
static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
@@ -361,19 +334,18 @@ static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
static inline void __i915_vma_pin(struct i915_vma *vma)
{
- vma->flags++;
- GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+ atomic_inc(&vma->pin_count);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
- vma->flags--;
+ atomic_dec(&vma->pin_count);
}
static inline void i915_vma_unpin(struct i915_vma *vma)
{
- GEM_BUG_ON(!i915_vma_is_pinned(vma));
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ GEM_BUG_ON(!atomic_read(&vma->pin_count));
__i915_vma_unpin(vma);
}
@@ -392,8 +364,6 @@ static inline bool i915_vma_is_bound(const struct i915_vma *vma,
* the caller must call i915_vma_unpin_iomap to relinquish the pinning
* after the iomapping is no longer required.
*
- * Callers must hold the struct_mutex.
- *
* Returns a valid iomapped pointer or ERR_PTR.
*/
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
@@ -405,8 +375,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
*
* Unpins the previously iomapped VMA from i915_vma_pin_iomap().
*
- * Callers must hold the struct_mutex. This function is only valid to be
- * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
+ * This function is only valid to be called on a VMA previously iomapped
+ * by the caller with i915_vma_pin_iomap().
*/
void i915_vma_unpin_iomap(struct i915_vma *vma);
@@ -416,6 +386,8 @@ static inline struct page *i915_vma_first_page(struct i915_vma *vma)
return sg_page(vma->pages->sgl);
}
+int __i915_vma_pin_fence(struct i915_vma *vma);
+
/**
* i915_vma_pin_fence - pin fencing state
* @vma: vma to pin fencing for
@@ -451,7 +423,6 @@ static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
static inline void
i915_vma_unpin_fence(struct i915_vma *vma)
{
- /* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */
if (vma->fence)
__i915_vma_unpin_fence(vma);
}
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index c3f44cbdb815..739c8df3f89d 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -2093,8 +2093,6 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
unsigned int pinctl;
u32 alignment;
- WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
alignment = intel_surf_alignment(fb, 0);
/* Note that the w/a also requires 64 PTE of padding following the
@@ -2175,13 +2173,9 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
-
- i915_gem_object_lock(vma->obj);
if (flags & PLANE_HAS_FENCE)
i915_vma_unpin_fence(vma);
i915_gem_object_unpin_from_display_plane(vma);
- i915_gem_object_unlock(vma->obj);
i915_vma_put(vma);
}
@@ -3048,7 +3042,6 @@ initial_plane_vma(struct drm_i915_private *i915,
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
u32 base, size;
- int err;
if (plane_config->size == 0)
return NULL;
@@ -3089,12 +3082,9 @@ initial_plane_vma(struct drm_i915_private *i915,
if (IS_ERR(vma))
return NULL;
- mutex_lock(&i915->drm.struct_mutex);
- err = i915_vma_pin(vma, 0, 0,
- PIN_GLOBAL | PIN_MAPPABLE |
- base | PIN_OFFSET_FIXED);
- mutex_unlock(&i915->drm.struct_mutex);
- if (err)
+ if (i915_vma_pin(vma, 0, 0,
+ PIN_GLOBAL | PIN_MAPPABLE |
+ base | PIN_OFFSET_FIXED))
goto err_vma;
if (i915_gem_object_is_tiled(vma->obj) &&
@@ -3280,14 +3270,12 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
intel_state->color_plane[0].stride =
intel_fb_pitch(fb, 0, intel_state->base.rotation);
- mutex_lock(&dev->struct_mutex);
intel_state->vma = i915_vma_get(plane_config->vma);
__i915_vma_pin(intel_state->vma);
if (intel_plane_uses_fence(intel_state) &&
i915_vma_pin_fence(intel_state->vma) == 0)
intel_state->flags |= PLANE_HAS_FENCE;
- vma->obj->pin_global++;
- mutex_unlock(&dev->struct_mutex);
+ atomic_inc(&vma->obj->pin_global);
plane_state->src_x = 0;
plane_state->src_y = 0;
@@ -14255,8 +14243,6 @@ static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
* bits. Some older platforms need special physical address handling for
* cursor planes.
*
- * Must be called with struct_mutex held.
- *
* Returns 0 on success, negative error code on failure.
*/
int
@@ -14313,15 +14299,8 @@ intel_prepare_plane_fb(struct drm_plane *plane,
if (ret)
return ret;
- ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
- if (ret) {
- i915_gem_object_unpin_pages(obj);
- return ret;
- }
-
ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
- mutex_unlock(&dev_priv->drm.struct_mutex);
i915_gem_object_unpin_pages(obj);
if (ret)
return ret;
@@ -14370,8 +14349,6 @@ intel_prepare_plane_fb(struct drm_plane *plane,
* @old_state: the state from the previous modeset
*
* Cleans up a framebuffer that has just been removed from a plane.
- *
- * Must be called with struct_mutex held.
*/
void
intel_cleanup_plane_fb(struct drm_plane *plane,
@@ -14387,9 +14364,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
}
/* Should only be called after a successful intel_prepare_plane_fb()! */
- mutex_lock(&dev_priv->drm.struct_mutex);
intel_plane_unpin_fb(to_intel_plane_state(old_state));
- mutex_unlock(&dev_priv->drm.struct_mutex);
}
int
@@ -14593,7 +14568,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
u32 src_w, u32 src_h,
struct drm_modeset_acquire_ctx *ctx)
{
- struct drm_i915_private *dev_priv = to_i915(crtc->dev);
struct drm_plane_state *old_plane_state, *new_plane_state;
struct intel_plane *intel_plane = to_intel_plane(plane);
struct intel_crtc_state *crtc_state =
@@ -14659,13 +14633,9 @@ intel_legacy_cursor_update(struct drm_plane *plane,
if (ret)
goto out_free;
- ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
- if (ret)
- goto out_free;
-
ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
if (ret)
- goto out_unlock;
+ goto out_free;
intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_FLIP);
intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->fb),
@@ -14695,8 +14665,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
-out_unlock:
- mutex_unlock(&dev_priv->drm.struct_mutex);
out_free:
if (new_crtc_state)
intel_crtc_destroy_state(crtc, &new_crtc_state->base);
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index c7c11d1842af..1464157d0ffe 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -212,7 +212,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
sizes->fb_height = intel_fb->base.height;
}
- mutex_lock(&dev->struct_mutex);
wakeref = intel_runtime_pm_get(dev_priv);
/* Pin the GGTT vma for our access via info->screen_base.
@@ -273,7 +272,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
ifbdev->vma_flags = flags;
intel_runtime_pm_put(dev_priv, wakeref);
- mutex_unlock(&dev->struct_mutex);
vga_switcheroo_client_fb_set(pdev, info);
return 0;
@@ -281,7 +279,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
intel_unpin_fb_vma(vma, flags);
out_unlock:
intel_runtime_pm_put(dev_priv, wakeref);
- mutex_unlock(&dev->struct_mutex);
return ret;
}
@@ -298,11 +295,8 @@ static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
drm_fb_helper_fini(&ifbdev->helper);
- if (ifbdev->vma) {
- mutex_lock(&ifbdev->helper.dev->struct_mutex);
+ if (ifbdev->vma)
intel_unpin_fb_vma(ifbdev->vma, ifbdev->vma_flags);
- mutex_unlock(&ifbdev->helper.dev->struct_mutex);
- }
if (ifbdev->fb)
drm_framebuffer_remove(&ifbdev->fb->base);
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 3b4e7686cd6c..184de18995e7 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -674,6 +674,7 @@ struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size)
return vma;
flags = PIN_GLOBAL | PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
+
ret = i915_vma_pin(vma, 0, 0, flags);
if (ret) {
i915_vma_put(vma);
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index cdc028220848..b969eef3331b 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -752,7 +752,6 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
struct i915_vma *vma;
int ret, tmp_width;
- lockdep_assert_held(&dev_priv->drm.struct_mutex);
WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
ret = intel_overlay_release_old_vid(overlay);
@@ -1304,22 +1303,16 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
struct i915_vma *vma;
int err;
- mutex_lock(&i915->drm.struct_mutex);
-
obj = i915_gem_object_create_stolen(i915, PAGE_SIZE);
if (obj == NULL)
obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
- if (IS_ERR(obj)) {
- err = PTR_ERR(obj);
- goto err_unlock;
- }
+ if (IS_ERR(obj))
+ return PTR_ERR(obj);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
i915_gem_object_put(obj);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_unlock;
- }
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
if (use_phys)
overlay->flip_addr = sg_dma_address(vma->obj->mm.pages->sgl);
@@ -1334,13 +1327,10 @@ static int get_registers(struct intel_overlay *overlay, bool use_phys)
}
overlay->reg_vma = vma;
- mutex_unlock(&i915->drm.struct_mutex);
return 0;
err_vma:
i915_vma_destroy(vma);
-err_unlock:
- mutex_unlock(&i915->drm.struct_mutex);
return err;
}
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 30b831408b7b..332ee5cb9120 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -338,7 +338,9 @@ static int igt_vma_pin1(void *arg)
if (!err) {
i915_vma_unpin(vma);
+ mutex_lock(&ggtt->vm.mutex);
err = i915_vma_unbind(vma);
+ mutex_unlock(&ggtt->vm.mutex);
if (err) {
pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
goto out;
--
2.20.1
More information about the Intel-gfx
mailing list