[PATCH 31/32] drm/i915: Move vma pinning under vm->mutex
Chris Wilson
chris at chris-wilson.co.uk
Fri Jun 22 19:46:12 UTC 2018
Use the local GTT lock (vm->mutex) for pinning the VMA (including
insertion and removal), as opposed to relying on the BKL struct_mutex.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/i915_gem_evict.c | 12 +++---
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 18 ++++----
drivers/gpu/drm/i915/i915_gem_stolen.c | 1 +
drivers/gpu/drm/i915/i915_vma.c | 41 +++++++++++--------
drivers/gpu/drm/i915/i915_vma.h | 41 ++++++++++++++-----
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 2 +-
6 files changed, 75 insertions(+), 40 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 8bc6fc66cea2..98635af0c9f7 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -132,7 +132,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
struct i915_vma *active;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict(vm, min_size, alignment, flags);
/*
@@ -247,7 +247,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
*/
list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
if (drm_mm_scan_remove_block(&scan, &vma->node))
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
else
list_del(&vma->evict_link);
}
@@ -291,7 +291,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
bool check_color;
int ret = 0;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
@@ -371,7 +371,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
* unbinding one vma from freeing (by dropping its active
* reference) another in our eviction list.
*/
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
@@ -402,7 +402,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
struct i915_vma *vma, *next;
int ret;
- lockdep_assert_held(&vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vm->mutex);
trace_i915_gem_evict_vm(vm);
/* Switch back to the default context in order to unpin
@@ -421,7 +421,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm)
if (i915_vma_is_pinned(vma))
continue;
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
list_add(&vma->evict_link, &eviction_list);
}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index c25b7b63f3f1..bff582a76d23 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -405,7 +405,7 @@ eb_pin_vma(struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
pin_flags |= PIN_GLOBAL;
- if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
+ if (unlikely(__i915_vma_pin(vma, 0, 0, pin_flags)))
return false;
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
@@ -619,9 +619,9 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
}
- err = i915_vma_pin(vma,
- entry->pad_to_size, entry->alignment,
- pin_flags);
+ err = __i915_vma_pin(vma,
+ entry->pad_to_size, entry->alignment,
+ pin_flags);
if (err)
return err;
@@ -633,7 +633,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
err = i915_vma_pin_fence(vma);
if (unlikely(err)) {
- i915_vma_unpin(vma);
+ __i915_vma_unpin(vma);
return err;
}
@@ -763,6 +763,7 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
batch = eb_batch_index(eb);
+ mutex_lock(&eb->vm->mutex);
for (i = 0; i < eb->buffer_count; i++) {
u32 handle = eb->exec[i].handle;
struct i915_lut_handle *lut;
@@ -816,13 +817,16 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
}
eb->args->flags |= __EXEC_VALIDATED;
- return eb_reserve(eb);
+ err = eb_reserve(eb);
+out_unlock:
+ mutex_unlock(&eb->vm->mutex);
+ return err;
err_obj:
i915_gem_object_put(obj);
err_vma:
eb->vma[i] = NULL;
- return err;
+ goto out_unlock;
}
static struct i915_vma *
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index b805abcd755b..175a5d499715 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -663,6 +663,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ mutex_lock(&vma->vm->mutex);
vma->pages = obj->mm.pages;
vma->flags |= I915_VMA_GLOBAL_BIND;
__i915_vma_set_map_and_fenceable(vma);
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index a96e1152cdd3..b5921b942324 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -289,6 +289,7 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
u32 vma_flags;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->size > vma->node.size);
@@ -332,8 +333,8 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
/* Access through the GTT requires the device to be awake. */
assert_rpm_wakelock_held(vma->vm->i915);
+ mutex_lock(&vma->vm->mutex);
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
err = -ENODEV;
goto err;
@@ -355,18 +356,20 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
vma->iomap = ptr;
}
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
- err = i915_vma_pin_fence(vma);
+ err = __i915_vma_pin_fence(vma);
if (err)
goto err_unpin;
i915_vma_set_ggtt_write(vma);
+ mutex_unlock(&vma->vm->mutex);
return ptr;
err_unpin:
__i915_vma_unpin(vma);
err:
+ mutex_unlock(&vma->vm->mutex);
return IO_ERR_PTR(err);
}
@@ -531,6 +534,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
u64 start, end;
int ret;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(i915_vma_is_closed(vma));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
@@ -643,9 +648,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, cache_level));
- mutex_lock(&vma->vm->mutex);
+ lockdep_assert_held(&vma->vm->mutex);
list_move_tail(&vma->vm_link, &vma->vm->bound_list);
- mutex_unlock(&vma->vm->mutex);
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
@@ -673,6 +677,8 @@ i915_vma_remove(struct i915_vma *vma)
{
struct drm_i915_private *i915 = vma->vm->i915;
+ lockdep_assert_held(&vma->vm->mutex);
+
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
@@ -711,7 +717,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
const unsigned int bound = vma->flags;
int ret;
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ lockdep_assert_held(&vma->vm->mutex);
GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
@@ -1029,7 +1035,7 @@ int i915_vma_unbind(struct i915_vma *vma)
* we currently hold, therefore it cannot free this object
* before we are finished).
*/
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
ret = i915_gem_active_retire(&vma->last_active,
&vma->vm->i915->drm.struct_mutex);
@@ -1060,15 +1066,18 @@ int i915_vma_unbind(struct i915_vma *vma)
}
GEM_BUG_ON(i915_vma_is_active(vma));
- if (i915_vma_is_pinned(vma))
- return -EBUSY;
+ mutex_lock(&vma->vm->mutex);
+ if (i915_vma_is_pinned(vma)) {
+ ret = -EBUSY;
+ goto out_unlock;
+ }
- if (!drm_mm_node_allocated(&vma->node))
- return 0;
+ if (!drm_mm_node_allocated(&vma->node)) {
+ ret = 0;
+ goto out_unlock;
+ }
if (i915_vma_is_map_and_fenceable(vma)) {
- mutex_lock(&vma->vm->mutex);
-
/*
* Check that we have flushed all writes through the GGTT
* before the unbind, other due to non-strict nature of those
@@ -1084,8 +1093,6 @@ int i915_vma_unbind(struct i915_vma *vma)
__i915_vma_iounmap(vma);
vma->flags &= ~I915_VMA_CAN_FENCE;
-
- mutex_unlock(&vma->vm->mutex);
}
GEM_BUG_ON(vma->fence);
GEM_BUG_ON(i915_vma_has_userfault(vma));
@@ -1098,7 +1105,9 @@ int i915_vma_unbind(struct i915_vma *vma)
i915_vma_remove(vma);
- return 0;
+out_unlock:
+ mutex_unlock(&vma->vm->mutex);
+ return ret;
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index b40d956849ed..72c0ee303eaf 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -276,11 +276,22 @@ void i915_vma_close(struct i915_vma *vma);
void i915_vma_reopen(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);
+static inline void ____i915_vma_pin(struct i915_vma *vma)
+{
+ lockdep_assert_held(&vma->vm->mutex);
+
+ vma->flags++;
+ GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+}
+
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
+
static inline int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+__i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
+ lockdep_assert_held(&vma->vm->mutex);
+
BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
@@ -297,32 +308,42 @@ i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
return __i915_vma_do_pin(vma, size, alignment, flags);
}
-static inline int i915_vma_pin_count(const struct i915_vma *vma)
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
- return vma->flags & I915_VMA_PIN_MASK;
+ int err;
+
+ mutex_lock(&vma->vm->mutex);
+ err = __i915_vma_pin(vma, size, alignment, flags);
+ mutex_unlock(&vma->vm->mutex);
+
+ return err;
}
-static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
+static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
- return i915_vma_pin_count(vma);
+ return READ_ONCE(vma->flags) & I915_VMA_PIN_MASK;
}
-static inline void __i915_vma_pin(struct i915_vma *vma)
+static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
{
- vma->flags++;
- GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+ return i915_vma_pin_count(vma);
}
static inline void __i915_vma_unpin(struct i915_vma *vma)
{
+ lockdep_assert_held(&vma->vm->mutex);
+ GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+
+ GEM_BUG_ON(!i915_vma_is_pinned(vma));
vma->flags--;
}
static inline void i915_vma_unpin(struct i915_vma *vma)
{
- GEM_BUG_ON(!i915_vma_is_pinned(vma));
- GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+ mutex_lock(&vma->vm->mutex);
__i915_vma_unpin(vma);
+ mutex_unlock(&vma->vm->mutex);
}
static inline bool i915_vma_is_bound(const struct i915_vma *vma,
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index da44c0f19a76..43ca405eec8b 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -1515,7 +1515,7 @@ static int igt_gtt_insert(void *arg)
goto out;
}
track_vma_bind(vma);
- __i915_vma_pin(vma);
+ ____i915_vma_pin(vma);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
}
--
2.18.0
More information about the Intel-gfx-trybot
mailing list