[PATCH 12/13] drm/i915: Rework intel_context pinning to do everything outside of pin_mutex
Maarten Lankhorst
maarten.lankhorst at linux.intel.com
Fri Jan 10 11:02:44 UTC 2020
Instead of doing everything inside of pin_mutex, we move all pinning
outside. Because i915_active has its own reference counting and
pinning is also having the same issues vs mutexes, we make sure
everything is pinned first, so the pinning in i915_active only needs
to bump refcounts. This allows us to take pin refcounts correctly
all the time.
Signed-off-by: Maarten Lankhorst <maarten.lankhorst at linux.intel.com>
---
drivers/gpu/drm/i915/gt/intel_context.c | 173 +++++++++++-------
drivers/gpu/drm/i915/gt/intel_context_types.h | 4 +-
drivers/gpu/drm/i915/gt/intel_lrc.c | 37 ++--
drivers/gpu/drm/i915/gt/intel_renderstate.c | 4 +-
.../gpu/drm/i915/gt/intel_ring_submission.c | 13 +-
drivers/gpu/drm/i915/gt/mock_engine.c | 13 +-
6 files changed, 157 insertions(+), 87 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index 9796a54b4f47..ca79bb65fa26 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -91,8 +91,75 @@ static void intel_context_active_release(struct intel_context *ce)
i915_active_release(&ce->active);
}
+static int __context_pin_state(struct i915_vma *vma)
+{
+ unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
+ int err;
+
+ err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH);
+ if (err)
+ return err;
+
+ /*
+ * And mark it as a globally pinned object to let the shrinker know
+ * it cannot reclaim the object until we release it.
+ */
+ i915_vma_make_unshrinkable(vma);
+ vma->obj->mm.dirty = true;
+
+ return 0;
+}
+
+static void __context_unpin_state(struct i915_vma *vma)
+{
+ i915_vma_make_shrinkable(vma);
+ __i915_vma_unpin(vma);
+}
+
+static int intel_context_pre_pin(struct intel_context *ce)
+{
+ int err;
+
+ CE_TRACE(ce, "active\n");
+
+ err = intel_ring_pin(ce->ring);
+ if (err)
+ return err;
+
+ err = intel_timeline_pin(ce->timeline);
+ if (err)
+ goto err_ring;
+
+ if (!ce->state)
+ return 0;
+
+ err = __context_pin_state(ce->state);
+ if (err)
+ goto err_timeline;
+
+ return 0;
+
+err_timeline:
+ intel_timeline_unpin(ce->timeline);
+err_ring:
+ intel_ring_unpin(ce->ring);
+ return err;
+}
+
+static void intel_context_post_unpin(struct intel_context *ce)
+{
+ if (ce->state)
+ __context_unpin_state(ce->state);
+
+ intel_timeline_unpin(ce->timeline);
+ intel_ring_unpin(ce->ring);
+}
+
int __intel_context_do_pin(struct intel_context *ce)
{
+ intel_wakeref_t wakeref;
+ bool handoff = false;
+ void *vaddr;
int err;
if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
@@ -101,34 +168,52 @@ int __intel_context_do_pin(struct intel_context *ce)
return err;
}
- if (mutex_lock_interruptible(&ce->pin_mutex))
- return -EINTR;
+ /*
+ * We always pin the context/ring/timeline here, to ensure a pin
+ * refcount for __intel_context_active(), which prevent a lock
+ * inversion of ce->pin_mutex vs dma_resv_lock().
+ */
+ err = intel_context_pre_pin(ce);
+ if (err)
+ goto err_wakeref;
+
+ err = ce->ops->pre_pin(ce, &vaddr);
+ if (err)
+ goto err_ctx_unpin;
+
+ err = mutex_lock_interruptible(&ce->pin_mutex);
+ if (err)
+ goto err_post_unpin;
if (likely(!atomic_read(&ce->pin_count))) {
err = intel_context_active_acquire(ce);
if (unlikely(err))
- goto err;
+ goto err_unlock;
- err = ce->ops->pin(ce);
- if (unlikely(err))
- goto err_active;
+ err = ce->ops->pin(ce, vaddr);
+ if (err) {
+ intel_context_active_release(ce);
+ goto err_unlock;
+ }
CE_TRACE(ce, "pin ring:{head:%04x, tail:%04x}\n",
ce->ring->head, ce->ring->tail);
+ handoff = true;
smp_mb__before_atomic(); /* flush pin before it is visible */
}
atomic_inc(&ce->pin_count);
GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
+err_unlock:
mutex_unlock(&ce->pin_mutex);
- return 0;
-
-err_active:
- intel_context_active_release(ce);
-err:
- mutex_unlock(&ce->pin_mutex);
+err_post_unpin:
+ if (!handoff)
+ ce->ops->post_unpin(ce);
+err_ctx_unpin:
+ intel_context_post_unpin(ce);
+err_wakeref:
return err;
}
@@ -139,6 +224,7 @@ void intel_context_unpin(struct intel_context *ce)
CE_TRACE(ce, "unpin\n");
ce->ops->unpin(ce);
+ ce->ops->post_unpin(ce);
/*
* Once released, we may asynchronously drop the active reference.
@@ -151,31 +237,6 @@ void intel_context_unpin(struct intel_context *ce)
intel_context_put(ce);
}
-static int __context_pin_state(struct i915_vma *vma)
-{
- unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
- int err;
-
- err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH);
- if (err)
- return err;
-
- /*
- * And mark it as a globally pinned object to let the shrinker know
- * it cannot reclaim the object until we release it.
- */
- i915_vma_make_unshrinkable(vma);
- vma->obj->mm.dirty = true;
-
- return 0;
-}
-
-static void __context_unpin_state(struct i915_vma *vma)
-{
- i915_vma_make_shrinkable(vma);
- __i915_vma_unpin(vma);
-}
-
__i915_active_call
static void __intel_context_retire(struct i915_active *active)
{
@@ -184,48 +245,24 @@ static void __intel_context_retire(struct i915_active *active)
CE_TRACE(ce, "retire\n");
set_bit(CONTEXT_VALID_BIT, &ce->flags);
- if (ce->state)
- __context_unpin_state(ce->state);
-
- intel_timeline_unpin(ce->timeline);
- intel_ring_unpin(ce->ring);
-
+ intel_context_post_unpin(ce);
intel_context_put(ce);
}
+__i915_active_call
static int __intel_context_active(struct i915_active *active)
{
struct intel_context *ce = container_of(active, typeof(*ce), active);
- int err;
-
- CE_TRACE(ce, "active\n");
intel_context_get(ce);
- err = intel_ring_pin(ce->ring);
- if (err)
- goto err_put;
-
- err = intel_timeline_pin(ce->timeline);
- if (err)
- goto err_ring;
-
- if (!ce->state)
- return 0;
-
- err = __context_pin_state(ce->state);
- if (err)
- goto err_timeline;
+ /* take an extra refcount on all ce->state, timeline and ring */
+ if (ce->state)
+ __i915_vma_pin(ce->state);
+ intel_timeline_pin(ce->timeline);
+ intel_ring_pin(ce->ring);
return 0;
-
-err_timeline:
- intel_timeline_unpin(ce->timeline);
-err_ring:
- intel_ring_unpin(ce->ring);
-err_put:
- intel_context_put(ce);
- return err;
}
void
diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h
index ca1420fb8b53..65457354eb39 100644
--- a/drivers/gpu/drm/i915/gt/intel_context_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_context_types.h
@@ -27,8 +27,10 @@ struct intel_ring;
struct intel_context_ops {
int (*alloc)(struct intel_context *ce);
- int (*pin)(struct intel_context *ce);
+ int (*pre_pin)(struct intel_context *ce, void **vaddr);
+ int (*pin)(struct intel_context *ce, void *vaddr);
void (*unpin)(struct intel_context *ce);
+ void (*post_unpin)(struct intel_context *ce);
void (*enter)(struct intel_context *ce);
void (*exit)(struct intel_context *ce);
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 9af1b2b493f4..401740e0ed42 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -2530,10 +2530,14 @@ static void execlists_context_unpin(struct intel_context *ce)
check_redzone((void *)ce->lrc_reg_state - LRC_STATE_PN * PAGE_SIZE,
ce->engine);
- i915_gem_object_unpin_map(ce->state->obj);
intel_ring_reset(ce->ring, ce->ring->tail);
}
+static void execlists_context_post_unpin(struct intel_context *ce)
+{
+ i915_gem_object_unpin_map(ce->state->obj);
+}
+
static void
__execlists_update_reg_state(const struct intel_context *ce,
const struct intel_engine_cs *engine)
@@ -2558,20 +2562,23 @@ __execlists_update_reg_state(const struct intel_context *ce,
}
static int
-__execlists_context_pin(struct intel_context *ce,
- struct intel_engine_cs *engine)
+execlists_context_pre_pin(struct intel_context *ce, void **vaddr)
{
- void *vaddr;
-
GEM_BUG_ON(!ce->state);
GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
- vaddr = i915_gem_object_pin_map(ce->state->obj,
- i915_coherent_map_type(engine->i915) |
+ *vaddr = i915_gem_object_pin_map(ce->state->obj,
+ i915_coherent_map_type(ce->engine->i915) |
I915_MAP_OVERRIDE);
- if (IS_ERR(vaddr))
- return PTR_ERR(vaddr);
+ return PTR_ERR_OR_ZERO(*vaddr);
+}
+
+static int
+__execlists_context_pin(struct intel_context *ce,
+ struct intel_engine_cs *engine,
+ void *vaddr)
+{
ce->lrc_desc = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE;
ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
__execlists_update_reg_state(ce, engine);
@@ -2579,9 +2586,9 @@ __execlists_context_pin(struct intel_context *ce,
return 0;
}
-static int execlists_context_pin(struct intel_context *ce)
+static int execlists_context_pin(struct intel_context *ce, void *vaddr)
{
- return __execlists_context_pin(ce, ce->engine);
+ return __execlists_context_pin(ce, ce->engine, vaddr);
}
static int execlists_context_alloc(struct intel_context *ce)
@@ -2623,8 +2630,10 @@ static void execlists_context_reset(struct intel_context *ce)
static const struct intel_context_ops execlists_context_ops = {
.alloc = execlists_context_alloc,
+ .pre_pin = execlists_context_pre_pin,
.pin = execlists_context_pin,
.unpin = execlists_context_unpin,
+ .post_unpin = execlists_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
@@ -4343,13 +4352,13 @@ static int virtual_context_alloc(struct intel_context *ce)
return __execlists_context_alloc(ce, ve->siblings[0]);
}
-static int virtual_context_pin(struct intel_context *ce)
+static int virtual_context_pin(struct intel_context *ce, void *vaddr)
{
struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
int err;
/* Note: we must use a real engine class for setting up reg state */
- err = __execlists_context_pin(ce, ve->siblings[0]);
+ err = __execlists_context_pin(ce, ve->siblings[0], vaddr);
if (err)
return err;
@@ -4382,8 +4391,10 @@ static void virtual_context_exit(struct intel_context *ce)
static const struct intel_context_ops virtual_context_ops = {
.alloc = virtual_context_alloc,
+ .pre_pin = execlists_context_pre_pin,
.pin = virtual_context_pin,
.unpin = execlists_context_unpin,
+ .post_unpin = execlists_context_post_unpin,
.enter = virtual_context_enter,
.exit = virtual_context_exit,
diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.c b/drivers/gpu/drm/i915/gt/intel_renderstate.c
index cb0b81fd918e..29481d9bf221 100644
--- a/drivers/gpu/drm/i915/gt/intel_renderstate.c
+++ b/drivers/gpu/drm/i915/gt/intel_renderstate.c
@@ -268,9 +268,7 @@ int intel_renderstate_emit(struct intel_renderstate *so,
void intel_renderstate_fini(struct intel_renderstate *so,
struct intel_context *ce)
{
- if (so->vma)
- i915_vma_unpin_and_release(&so->vma, 0);
-
+ i915_vma_unpin_and_release(&so->vma, 0);
intel_context_unpin(ce);
i915_gem_ww_ctx_fini(&so->ww);
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index bc44fe8e5ffa..e3714666b013 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -1230,6 +1230,10 @@ static void __context_unpin_ppgtt(struct intel_context *ce)
}
static void ring_context_unpin(struct intel_context *ce)
+{
+}
+
+static void ring_context_post_unpin(struct intel_context *ce)
{
__context_unpin_ppgtt(ce);
}
@@ -1327,11 +1331,16 @@ static int ring_context_alloc(struct intel_context *ce)
return 0;
}
-static int ring_context_pin(struct intel_context *ce)
+static int ring_context_pre_pin(struct intel_context *ce, void **unused)
{
return __context_pin_ppgtt(ce);
}
+static int ring_context_pin(struct intel_context *ce, void *unused)
+{
+ return 0;
+}
+
static void ring_context_reset(struct intel_context *ce)
{
intel_ring_reset(ce->ring, ce->ring->emit);
@@ -1340,8 +1349,10 @@ static void ring_context_reset(struct intel_context *ce)
static const struct intel_context_ops ring_context_ops = {
.alloc = ring_context_alloc,
+ .pre_pin = ring_context_pre_pin,
.pin = ring_context_pin,
.unpin = ring_context_unpin,
+ .post_unpin = ring_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c
index a560b7eee2cd..28c8de21e5ee 100644
--- a/drivers/gpu/drm/i915/gt/mock_engine.c
+++ b/drivers/gpu/drm/i915/gt/mock_engine.c
@@ -114,6 +114,10 @@ static void mock_context_unpin(struct intel_context *ce)
{
}
+static void mock_context_post_unpin(struct intel_context *ce)
+{
+}
+
static void mock_context_destroy(struct kref *ref)
{
struct intel_context *ce = container_of(ref, typeof(*ce), ref);
@@ -147,7 +151,12 @@ static int mock_context_alloc(struct intel_context *ce)
return 0;
}
-static int mock_context_pin(struct intel_context *ce)
+static int mock_context_pre_pin(struct intel_context *ce, void **unused)
+{
+ return 0;
+}
+
+static int mock_context_pin(struct intel_context *ce, void *unused)
{
return 0;
}
@@ -159,8 +168,10 @@ static void mock_context_reset(struct intel_context *ce)
static const struct intel_context_ops mock_context_ops = {
.alloc = mock_context_alloc,
+ .pre_pin = mock_context_pre_pin,
.pin = mock_context_pin,
.unpin = mock_context_unpin,
+ .post_unpin = mock_context_post_unpin,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
--
2.24.1
More information about the Intel-gfx-trybot
mailing list