[PATCH 45/45] drm/i915: Move object close under its own lock
Chris Wilson
chris at chris-wilson.co.uk
Mon Mar 4 10:42:36 UTC 2019
Use i915_gem_object_lock() to guard the LUT and active reference to
allow us to break free of struct_mutex for handling GEM_CLOSE.
Testcase: igt/gem_close_race
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gem/i915_gem_context.c | 24 ++++++++-
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 23 +++++++--
drivers/gpu/drm/i915/gem/i915_gem_object.c | 49 ++++++++++++------
.../gpu/drm/i915/gem/i915_gem_object_types.h | 1 +
drivers/gpu/drm/i915/i915_drv.h | 4 +-
drivers/gpu/drm/i915/i915_gem.c | 1 +
drivers/gpu/drm/i915/i915_vma.c | 50 +++++++++++++------
drivers/gpu/drm/i915/i915_vma.h | 15 +++---
.../gpu/drm/i915/selftests/mock_gem_device.c | 1 +
9 files changed, 123 insertions(+), 45 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 909134b0f007..5fac54a76704 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -114,8 +114,24 @@ static void lut_close(struct i915_gem_context *ctx)
struct radix_tree_iter iter;
void __rcu **slot;
+ lockdep_assert_held(&ctx->mutex);
+
list_for_each_entry_safe(lut, ln, &ctx->handles_list, ctx_link) {
+ struct drm_i915_gem_object *obj = lut->obj;
+
+ i915_gem_object_lock(obj);
+
list_del(&lut->obj_link);
+
+ if (!i915_gem_object_has_active_reference(obj) &&
+ i915_gem_object_is_active(obj)) {
+ i915_gem_object_get(obj);
+ i915_gem_object_set_active_reference(obj);
+ }
+
+ i915_gem_object_unlock(obj);
+ i915_gem_object_put(obj);
+
i915_lut_handle_free(lut);
}
@@ -124,9 +140,7 @@ static void lut_close(struct i915_gem_context *ctx)
struct i915_vma *vma = rcu_dereference_raw(*slot);
radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
-
vma->open_count--;
- __i915_gem_object_release_unless_active(vma->obj);
}
rcu_read_unlock();
}
@@ -309,6 +323,8 @@ void i915_gem_context_release(struct kref *ref)
static void context_close(struct i915_gem_context *ctx)
{
+ mutex_lock(&ctx->mutex);
+
i915_gem_context_set_closed(ctx);
/*
@@ -327,6 +343,8 @@ static void context_close(struct i915_gem_context *ctx)
i915_ppgtt_close(ctx->ppgtt);
ctx->file_priv = ERR_PTR(-EBADF);
+
+ mutex_unlock(&ctx->mutex);
i915_gem_context_put(ctx);
}
@@ -1041,7 +1059,9 @@ static int set_ppgtt(struct i915_gem_context *ctx,
goto unlock;
/* Teardown the existing obj:vma cache, it will have to be rebuilt. */
+ mutex_lock(&ctx->mutex);
lut_close(ctx);
+ mutex_unlock(&ctx->mutex);
old = __set_ppgtt(ctx, ppgtt);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index e65ed2049671..eb979066e6bb 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -803,9 +803,6 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
unsigned int i, batch;
int err;
- if (unlikely(i915_gem_context_is_closed(eb->ctx)))
- return -ENOENT;
-
if (unlikely(i915_gem_context_is_banned(eb->ctx)))
return -EIO;
@@ -814,6 +811,12 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
batch = eb_batch_index(eb);
+ mutex_lock(&eb->ctx->mutex);
+ if (unlikely(i915_gem_context_is_closed(eb->ctx))) {
+ err = -ENOENT;
+ goto err_ctx;
+ }
+
for (i = 0; i < eb->buffer_count; i++) {
u32 handle = eb->exec[i].handle;
struct i915_lut_handle *lut;
@@ -850,10 +853,16 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
/* transfer ref to ctx */
if (!vma->open_count++)
i915_vma_reopen(vma);
- list_add(&lut->obj_link, &obj->lut_list);
+
+ lut->handle = handle;
+
list_add(&lut->ctx_link, &eb->ctx->handles_list);
lut->ctx = eb->ctx;
- lut->handle = handle;
+
+ i915_gem_object_lock(obj);
+ lut->obj = obj;
+ list_add(&lut->obj_link, &obj->lut_list);
+ i915_gem_object_unlock(obj);
add_vma:
err = eb_add_vma(eb, i, batch, vma);
@@ -866,6 +875,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
eb_vma_misplaced(&eb->exec[i], vma, eb->flags[i]));
}
+ mutex_unlock(&eb->ctx->mutex);
+
eb->args->flags |= __EXEC_VALIDATED;
return eb_reserve(eb);
@@ -873,6 +884,8 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
i915_gem_object_put(obj);
err_vma:
eb->vma[i] = NULL;
+err_ctx:
+ mutex_unlock(&eb->ctx->mutex);
return err;
}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 28cf9b037d29..6032c079674f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -125,12 +125,12 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
{
- struct drm_i915_private *i915 = to_i915(gem->dev);
struct drm_i915_gem_object *obj = to_intel_bo(gem);
struct drm_i915_file_private *fpriv = file->driver_priv;
struct i915_lut_handle *lut, *ln;
- mutex_lock(&i915->drm.struct_mutex);
+ i915_gem_object_get(obj);
+ i915_gem_object_lock(obj);
list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
struct i915_gem_context *ctx = lut->ctx;
@@ -139,25 +139,43 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
if (ctx->file_priv != fpriv)
continue;
-
- vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
- GEM_BUG_ON(vma->obj != obj);
-
- /* We allow the process to have multiple handles to the same
+ /*
+ * We allow the process to have multiple handles to the same
* vma, in the same fd namespace, by virtue of flink/open.
*/
GEM_BUG_ON(!vma->open_count);
- if (!--vma->open_count && !i915_vma_is_ggtt(vma))
+ if (!--vma->open_count && !i915_vma_is_ggtt(vma)) {
+ mutex_lock(&obj->base.dev->struct_mutex);
i915_vma_close(vma);
+ mutex_unlock(&obj->base.dev->struct_mutex);
+ }
list_del(&lut->obj_link);
+ i915_gem_object_unlock(obj);
+
+ mutex_lock(&ctx->mutex);
+
+ vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
+ GEM_BUG_ON(vma->obj != obj);
list_del(&lut->ctx_link);
+ mutex_unlock(&ctx->mutex);
+
i915_lut_handle_free(lut);
- __i915_gem_object_release_unless_active(obj);
+ i915_gem_object_put(obj);
+
+ i915_gem_object_lock(obj);
+ list_safe_reset_next(lut, ln, obj_link);
+ }
+
+ if (!i915_gem_object_has_active_reference(obj) &&
+ i915_gem_object_is_active(obj)) {
+ i915_gem_object_get(obj);
+ i915_gem_object_set_active_reference(obj);
}
- mutex_unlock(&i915->drm.struct_mutex);
+ i915_gem_object_unlock(obj);
+ i915_gem_object_put(obj);
}
static bool discard_backing_storage(struct drm_i915_gem_object *obj)
@@ -348,13 +366,16 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
{
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ i915_gem_object_lock(obj);
if (!i915_gem_object_has_active_reference(obj) &&
- i915_gem_object_is_active(obj))
+ i915_gem_object_is_active(obj)) {
+ i915_gem_object_get(obj);
i915_gem_object_set_active_reference(obj);
- else
- i915_gem_object_put(obj);
+ }
+
+ i915_gem_object_unlock(obj);
+ i915_gem_object_put(obj);
}
static inline enum fb_op_origin
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index da6a33e2395f..3ac15ffce560 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -26,6 +26,7 @@ struct i915_lut_handle {
struct list_head obj_link;
struct list_head ctx_link;
struct i915_gem_context *ctx;
+ struct drm_i915_gem_object *obj;
u32 handle;
};
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 30a47fa64f17..7a3a629913ad 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1983,10 +1983,12 @@ struct drm_i915_private {
} timelines;
struct list_head active_rings;
- struct list_head closed_vma;
unsigned long active_engines;
u32 active_requests;
+ struct list_head closed_vma;
+ spinlock_t closed_lock; /* guards the list of closed_vma */
+
/**
* Is the GPU currently considered idle, or busy executing
* userspace requests? Whilst idle, we allow runtime power
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index b61bba8f6fd6..852f112031ed 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2123,6 +2123,7 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
INIT_LIST_HEAD(&dev_priv->gt.active_rings);
INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
+ spin_lock_init(&dev_priv->gt.closed_lock);
i915_gem_init__mm(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index bb82374d107f..5fdfe2547fc8 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -113,7 +113,13 @@ static void __i915_vma_retire(struct i915_active *ref)
obj_bump_mru(obj);
if (i915_gem_object_has_active_reference(obj)) {
- i915_gem_object_clear_active_reference(obj);
+ i915_gem_object_lock(obj);
+ i915_gem_object_get(obj);
+ if (i915_gem_object_has_active_reference(obj)) {
+ i915_gem_object_clear_active_reference(obj);
+ i915_gem_object_put(obj);
+ }
+ i915_gem_object_unlock(obj);
i915_gem_object_put(obj);
}
}
@@ -143,6 +149,8 @@ vma_create(struct drm_i915_gem_object *obj,
vma->size = obj->base.size;
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
+ INIT_LIST_HEAD(&vma->closed_link);
+
if (view && view->type != I915_GGTT_VIEW_NORMAL) {
vma->ggtt_view = *view;
if (view->type == I915_GGTT_VIEW_PARTIAL) {
@@ -785,10 +793,9 @@ int __i915_vma_do_pin(struct i915_vma *vma,
void i915_vma_close(struct i915_vma *vma)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ struct drm_i915_private *i915 = vma->vm->i915;
GEM_BUG_ON(i915_vma_is_closed(vma));
- vma->flags |= I915_VMA_CLOSED;
/*
* We defer actually closing, unbinding and destroying the VMA until
@@ -802,17 +809,26 @@ void i915_vma_close(struct i915_vma *vma)
* causing us to rebind the VMA once more. This ends up being a lot
* of wasted work for the steady state.
*/
- list_add_tail(&vma->closed_link, &vma->vm->i915->gt.closed_vma);
+ spin_lock(&i915->gt.closed_lock);
+ list_add_tail(&vma->closed_link, &i915->gt.closed_vma);
+ spin_unlock(&i915->gt.closed_lock);
}
-void i915_vma_reopen(struct i915_vma *vma)
+static void __i915_vma_remove_closed(struct i915_vma *vma)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ struct drm_i915_private *i915 = vma->vm->i915;
- if (vma->flags & I915_VMA_CLOSED) {
- vma->flags &= ~I915_VMA_CLOSED;
- list_del(&vma->closed_link);
- }
+ if (!i915_vma_is_closed(vma))
+ return;
+
+ spin_lock(&i915->gt.closed_lock);
+ list_del_init(&vma->closed_link);
+ spin_unlock(&i915->gt.closed_lock);
+}
+
+void i915_vma_reopen(struct i915_vma *vma)
+{
+ __i915_vma_remove_closed(vma);
}
static void __i915_vma_destroy(struct i915_vma *vma)
@@ -846,8 +862,7 @@ void i915_vma_destroy(struct i915_vma *vma)
GEM_BUG_ON(i915_vma_is_pinned(vma));
- if (i915_vma_is_closed(vma))
- list_del(&vma->closed_link);
+ __i915_vma_remove_closed(vma);
WARN_ON(i915_vma_unbind(vma));
GEM_BUG_ON(i915_vma_is_active(vma));
@@ -859,12 +874,17 @@ void i915_vma_parked(struct drm_i915_private *i915)
{
struct i915_vma *vma, *next;
+ spin_lock(&i915->gt.closed_lock);
list_for_each_entry_safe(vma, next, &i915->gt.closed_vma, closed_link) {
- GEM_BUG_ON(!i915_vma_is_closed(vma));
+ list_del_init(&vma->closed_link);
+ spin_unlock(&i915->gt.closed_lock);
+
i915_vma_destroy(vma);
- }
- GEM_BUG_ON(!list_empty(&i915->gt.closed_vma));
+ spin_lock(&i915->gt.closed_lock);
+ list_safe_reset_next(vma, next, closed_link);
+ }
+ spin_unlock(&i915->gt.closed_lock);
}
static void __i915_vma_iounmap(struct i915_vma *vma)
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index b0fb1714decf..894c6986d58c 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -104,10 +104,9 @@ struct i915_vma {
#define I915_VMA_GGTT BIT(11)
#define I915_VMA_CAN_FENCE BIT(12)
-#define I915_VMA_CLOSED BIT(13)
-#define I915_VMA_USERFAULT_BIT 14
+#define I915_VMA_USERFAULT_BIT 13
#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT)
-#define I915_VMA_GGTT_WRITE BIT(15)
+#define I915_VMA_GGTT_WRITE BIT(14)
struct i915_active active;
struct i915_active_request last_fence;
@@ -190,11 +189,6 @@ static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
return vma->flags & I915_VMA_CAN_FENCE;
}
-static inline bool i915_vma_is_closed(const struct i915_vma *vma)
-{
- return vma->flags & I915_VMA_CLOSED;
-}
-
static inline bool i915_vma_set_userfault(struct i915_vma *vma)
{
GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
@@ -211,6 +205,11 @@ static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
}
+static inline bool i915_vma_is_closed(const struct i915_vma *vma)
+{
+ return !list_empty(&vma->closed_link);
+}
+
static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
{
GEM_BUG_ON(!i915_vma_is_ggtt(vma));
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 930f9cb1661f..37c9a5706f69 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -206,6 +206,7 @@ struct drm_i915_private *mock_gem_device(void)
INIT_LIST_HEAD(&i915->gt.active_rings);
INIT_LIST_HEAD(&i915->gt.closed_vma);
+ spin_lock_init(&i915->gt.closed_lock);
mutex_lock(&i915->drm.struct_mutex);
--
2.20.1
More information about the Intel-gfx-trybot
mailing list