[PATCH 13/15] drm/i915/gt: Implement ring scheduler for gen6/gen7
Chris Wilson
chris at chris-wilson.co.uk
Sun May 31 14:28:14 UTC 2020
A key problem we have with the legacy ring buffer submission is that it
only allows for FIFO queuing. All clients share the global request queue
and must contend for its lock when submitting. As any client may need to
wait for external events, all clients must then wait. However, if we
stage each client into their own virtual ringbuffer with their own
timelines, we can copy the client requests into the global ringbuffer
only when they are ready, reordering the submission around stalls.
Furthermore, the ability to reorder gives us rudimentarily priority
sorting -- although without preemption support, once something is on the
GPU it stays on the GPU, and so it is still possible for a hog to delay
a high priority request (such as updating the display). However, it does
means that in keeping a short submission queue, the high priority
request will be next.
The implementation uses the MI_USER_INTERRUPT at the end of every
request to track completion, so is more interrupt happy than execlists
[which has an interrupt for each context event, albeit two]. Our
interrupts on these system are relatively heavy, and in the past we have
been able to completely starve Sandybrige by the interrupt traffic. Our
interrupt handlers are being much better (in part offloading the work to
bottom halves leaving the interrupt itself only dealing with acking the
registers) but we can still see the impact of starvation in the uneven
submission latency on a saturated system.
Another side effect of hooking up a tasklet to every CS interrupt is
that we should get effective soft-rc6, which from our experience with
execlist scheduling, we expect to respond faster than the HW EI. This
also implies we need to enable the RPS timer, and for that we will need
busy-stats.
This virtual execlists scheme should be applicable to all older
platforms, only currently implemented for gen6/7 for expediency and
testing. Sandybridge, Ivybridge, Bayrail and Haswell all also benefit
from having client timelines to enable semaphores for inter-engine
synchronisation.
Overall though, the short sumission queues and extra interrupts do not
appear to be affecting throughput (+-10%, some tasks even improve to the
reduced request overheads) and improve latency. [Which is a massive
improvement since the introduction of Sandybridge!]
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
.../gpu/drm/i915/gt/intel_ring_scheduler.c | 428 +++++++++++++++++-
.../drm/i915/gt/selftest_ring_submission.c | 3 +
2 files changed, 425 insertions(+), 6 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c b/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
index 0d65009e4473..f9750023261e 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
@@ -7,6 +7,10 @@
#include <drm/i915_drm.h>
+#include "gen2_engine_cs.h"
+#include "gen6_engine_cs.h"
+#include "gen6_ppgtt.h"
+#include "gen7_renderclear.h"
#include "i915_drv.h"
#include "intel_context.h"
#include "intel_engine_stats.h"
@@ -282,8 +286,263 @@ static void ring_copy(struct intel_ring *dst,
memcpy(out, src->vaddr + start, end - start);
}
+static void mi_set_context(struct intel_ring *ring,
+ struct intel_engine_cs *engine,
+ struct intel_context *ce,
+ u32 flags)
+{
+ struct drm_i915_private *i915 = engine->i915;
+ enum intel_engine_id id;
+ const int num_engines =
+ IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0;
+ int len;
+ u32 *cs;
+
+ len = 4;
+ if (IS_GEN(i915, 7))
+ len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
+ else if (IS_GEN(i915, 5))
+ len += 2;
+
+ cs = ring_map_dw(ring, len);
+
+ /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
+ if (IS_GEN(i915, 7)) {
+ *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+ if (num_engines) {
+ struct intel_engine_cs *signaller;
+
+ *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
+ for_each_engine(signaller, engine->gt, id) {
+ if (signaller == engine)
+ continue;
+
+ *cs++ = i915_mmio_reg_offset(
+ RING_PSMI_CTL(signaller->mmio_base));
+ *cs++ = _MASKED_BIT_ENABLE(
+ GEN6_PSMI_SLEEP_MSG_DISABLE);
+ }
+ }
+ } else if (IS_GEN(i915, 5)) {
+ /*
+ * This w/a is only listed for pre-production ilk a/b steppings,
+ * but is also mentioned for programming the powerctx. To be
+ * safe, just apply the workaround; we do not use SyncFlush so
+ * this should never take effect and so be a no-op!
+ */
+ *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
+ }
+
+ *cs++ = MI_NOOP;
+ *cs++ = MI_SET_CONTEXT;
+ *cs++ = i915_ggtt_offset(ce->state) | flags;
+ /*
+ * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
+ * WaMiSetContext_Hang:snb,ivb,vlv
+ */
+ *cs++ = MI_NOOP;
+
+ if (IS_GEN(i915, 7)) {
+ if (num_engines) {
+ struct intel_engine_cs *signaller;
+ i915_reg_t last_reg = {}; /* keep gcc quiet */
+
+ *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
+ for_each_engine(signaller, engine->gt, id) {
+ if (signaller == engine)
+ continue;
+
+ last_reg = RING_PSMI_CTL(signaller->mmio_base);
+ *cs++ = i915_mmio_reg_offset(last_reg);
+ *cs++ = _MASKED_BIT_DISABLE(
+ GEN6_PSMI_SLEEP_MSG_DISABLE);
+ }
+
+ /* Insert a delay before the next switch! */
+ *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
+ *cs++ = i915_mmio_reg_offset(last_reg);
+ *cs++ = intel_gt_scratch_offset(engine->gt,
+ INTEL_GT_SCRATCH_FIELD_DEFAULT);
+ *cs++ = MI_NOOP;
+ }
+ *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
+ } else if (IS_GEN(i915, 5)) {
+ *cs++ = MI_SUSPEND_FLUSH;
+ }
+}
+
+static struct i915_address_space *vm_alias(struct i915_address_space *vm)
+{
+ if (i915_is_ggtt(vm))
+ vm = &i915_vm_to_ggtt(vm)->alias->vm;
+
+ return vm;
+}
+
+static void load_pd_dir(struct intel_ring *ring,
+ struct intel_engine_cs *engine,
+ const struct i915_ppgtt *ppgtt)
+{
+ u32 *cs = ring_map_dw(ring, 12);
+
+ *cs++ = MI_LOAD_REGISTER_IMM(1);
+ *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
+ *cs++ = PP_DIR_DCLV_2G;
+
+ *cs++ = MI_LOAD_REGISTER_IMM(1);
+ *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
+ *cs++ = px_base(ppgtt->pd)->ggtt_offset << 10;
+
+ /* Stall until the page table load is complete? */
+ *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
+ *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
+ *cs++ = intel_gt_scratch_offset(engine->gt,
+ INTEL_GT_SCRATCH_FIELD_DEFAULT);
+
+ *cs++ = MI_LOAD_REGISTER_IMM(1);
+ *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
+ *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
+}
+
+static struct i915_address_space *current_vm(struct intel_engine_cs *engine)
+{
+ struct intel_context *old = engine->legacy.context;
+
+ return old ? vm_alias(old->vm) : NULL;
+}
+
+static void gen6_emit_invalidate_rcs(struct intel_ring *ring,
+ struct intel_engine_cs *engine)
+{
+ u32 addr, flags;
+ u32 *cs;
+
+ addr = intel_gt_scratch_offset(engine->gt,
+ INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
+
+ flags = PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
+ flags |= PIPE_CONTROL_TLB_INVALIDATE;
+
+ if (INTEL_GEN(engine->i915) >= 7)
+ flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
+ else
+ addr |= PIPE_CONTROL_GLOBAL_GTT;
+
+ cs = ring_map_dw(ring, 4);
+ *cs++ = GFX_OP_PIPE_CONTROL(4);
+ *cs++ = flags;
+ *cs++ = addr;
+ *cs++ = 0;
+}
+
+static struct i915_address_space *
+clear_residuals(struct intel_ring *ring, struct intel_engine_cs *engine)
+{
+ struct intel_context *ce = engine->kernel_context;
+ struct i915_address_space *vm = vm_alias(ce->vm);
+ u32 flags;
+
+ if (vm != current_vm(engine))
+ load_pd_dir(ring, engine, i915_vm_to_ppgtt(vm));
+
+ if (ce->state)
+ mi_set_context(ring, engine, ce,
+ MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT);
+
+ if (IS_HASWELL(engine->i915))
+ flags = MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW;
+ else
+ flags = MI_BATCH_NON_SECURE_I965;
+
+ __gen6_emit_bb_start(ring_map_dw(ring, 2),
+ engine->wa_ctx.vma->node.start, flags);
+
+ gen6_emit_invalidate_rcs(ring, engine);
+
+ return vm;
+}
+
+static void remap_l3_slice(struct intel_ring *ring,
+ struct intel_engine_cs *engine,
+ int slice)
+{
+ u32 *cs, *remap_info = engine->i915->l3_parity.remap_info[slice];
+ int i;
+
+ if (!remap_info)
+ return;
+
+ /*
+ * Note: We do not worry about the concurrent register cacheline hang
+ * here because no other code should access these registers other than
+ * at initialization time.
+ */
+ cs = ring_map_dw(ring, GEN7_L3LOG_SIZE / 4 * 2 + 2);
+ *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE / 4);
+ for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
+ *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
+ *cs++ = remap_info[i];
+ }
+ *cs++ = MI_NOOP;
+}
+
+static void remap_l3(struct intel_ring *ring,
+ struct intel_engine_cs *engine,
+ struct intel_context *ce)
+{
+ struct i915_gem_context *ctx =
+ rcu_dereference_protected(ce->gem_context, true);
+ int bit, idx = -1;
+
+ if (!ctx || !ctx->remap_slice)
+ return;
+
+ do {
+ bit = ffs(ctx->remap_slice);
+ remap_l3_slice(ring, engine, idx += bit);
+ } while (ctx->remap_slice >>= bit);
+}
+
static void switch_context(struct intel_ring *ring, struct i915_request *rq)
{
+ struct intel_engine_cs *engine = rq->engine;
+ struct i915_address_space *cvm = current_vm(engine);
+ struct intel_context *ce = rq->context;
+ struct i915_address_space *vm;
+
+ if (engine->wa_ctx.vma && ce != engine->kernel_context) {
+ if (engine->wa_ctx.vma->private != ce) {
+ cvm = clear_residuals(ring, engine);
+ intel_context_put(engine->wa_ctx.vma->private);
+ engine->wa_ctx.vma->private = intel_context_get(ce);
+ }
+ }
+
+ vm = vm_alias(ce->vm);
+ if (vm != cvm)
+ load_pd_dir(ring, engine, i915_vm_to_ppgtt(vm));
+
+ if (ce->state) {
+ u32 flags;
+
+ GEM_BUG_ON(engine->id != RCS0);
+
+ /* For resource streamer on HSW+ and power context elsewhere */
+ BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN);
+ BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN);
+
+ flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT;
+ if (test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
+ gen6_emit_invalidate_rcs(ring, engine);
+ flags |= MI_RESTORE_EXT_STATE_EN;
+ } else {
+ flags |= MI_RESTORE_INHIBIT;
+ }
+
+ mi_set_context(ring, engine, ce, flags);
+ }
+
+ remap_l3(ring, engine, ce);
}
static struct i915_request *ring_submit(struct i915_request *rq)
@@ -453,6 +712,33 @@ static void submission_unpark(struct intel_engine_cs *engine)
intel_engine_pin_breadcrumbs_irq(engine);
}
+static int gen6_emit_init_breadcrumb(struct i915_request *rq)
+{
+ struct intel_timeline *tl = i915_request_timeline(rq);
+ u32 *cs;
+
+ GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq));
+ if (!tl->has_initial_breadcrumb)
+ return 0;
+
+ cs = intel_ring_begin(rq, 4);
+ if (IS_ERR(cs))
+ return PTR_ERR(cs);
+
+ *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+ *cs++ = 0;
+ *cs++ = tl->hwsp_offset;
+ *cs++ = rq->fence.seqno - 1;
+
+ intel_ring_advance(rq, cs);
+
+ /* Record the updated position of the request's payload */
+ rq->infix = intel_ring_offset(rq, cs);
+
+ __set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags);
+ return 0;
+}
+
static void ring_context_destroy(struct kref *ref)
{
struct intel_context *ce = container_of(ref, typeof(*ce), ref);
@@ -468,6 +754,32 @@ static void ring_context_destroy(struct kref *ref)
intel_context_free(ce);
}
+static int __context_pin_ppgtt(struct intel_context *ce)
+{
+ struct i915_address_space *vm;
+ int err = 0;
+
+ vm = vm_alias(ce->vm);
+ if (vm)
+ err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)));
+
+ return err;
+}
+
+static void __context_unpin_ppgtt(struct intel_context *ce)
+{
+ struct i915_address_space *vm;
+
+ vm = vm_alias(ce->vm);
+ if (vm)
+ gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
+}
+
+static void ring_context_unpin(struct intel_context *ce)
+{
+ __context_unpin_ppgtt(ce);
+}
+
static struct i915_vma *
alloc_context_vma(struct intel_engine_cs *engine)
{
@@ -593,13 +905,9 @@ static int ring_context_alloc(struct intel_context *ce)
return err;
}
-static void ring_context_unpin(struct intel_context *ce)
-{
-}
-
static int ring_context_pin(struct intel_context *ce)
{
- return 0;
+ return __context_pin_ppgtt(ce);
}
static void ring_context_reset(struct intel_context *ce)
@@ -659,12 +967,19 @@ static void ring_release(struct intel_engine_cs *engine)
set_current_context(&engine->legacy.context, NULL);
+ if (engine->wa_ctx.vma) {
+ intel_context_put(engine->wa_ctx.vma->private);
+ i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
+ }
+
intel_ring_unpin(engine->legacy.ring);
intel_ring_put(engine->legacy.ring);
}
static void setup_irq(struct intel_engine_cs *engine)
{
+ engine->irq_enable = gen6_irq_enable;
+ engine->irq_disable = gen6_irq_disable;
}
static void setup_common(struct intel_engine_cs *engine)
@@ -673,7 +988,7 @@ static void setup_common(struct intel_engine_cs *engine)
/* gen8+ are only supported with execlists */
GEM_BUG_ON(INTEL_GEN(i915) >= 8);
- GEM_BUG_ON(INTEL_GEN(i915) < 8);
+ GEM_BUG_ON(INTEL_GEN(i915) < 6);
setup_irq(engine);
@@ -686,25 +1001,64 @@ static void setup_common(struct intel_engine_cs *engine)
engine->cops = &ring_context_ops;
engine->request_alloc = ring_request_alloc;
+ engine->emit_init_breadcrumb = gen6_emit_init_breadcrumb;
+ if (INTEL_GEN(i915) >= 7)
+ engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
+ else if (INTEL_GEN(i915) >= 6)
+ engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
+ else
+ engine->emit_fini_breadcrumb = gen3_emit_breadcrumb;
+
engine->set_default_submission = set_default_submission;
+
+ engine->emit_bb_start = gen6_emit_bb_start;
}
static void setup_rcs(struct intel_engine_cs *engine)
{
+ struct drm_i915_private *i915 = engine->i915;
+
+ if (HAS_L3_DPF(i915))
+ engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
+
+ engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
+
+ if (INTEL_GEN(i915) >= 7) {
+ engine->emit_flush = gen7_emit_flush_rcs;
+ engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs;
+ if (IS_HASWELL(i915))
+ engine->emit_bb_start = hsw_emit_bb_start;
+ } else {
+ engine->emit_flush = gen6_emit_flush_rcs;
+ engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs;
+ }
+
engine->resume = intel_ring_submission_resume_rcs;
}
static void setup_vcs(struct intel_engine_cs *engine)
{
+ engine->emit_flush = gen6_emit_flush_vcs;
+ engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
+
+ if (IS_GEN(engine->i915, 6))
+ engine->fw_domain = FORCEWAKE_ALL;
}
static void setup_bcs(struct intel_engine_cs *engine)
{
+ engine->emit_flush = gen6_emit_flush_xcs;
+ engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
}
static void setup_vecs(struct intel_engine_cs *engine)
{
GEM_BUG_ON(!IS_HASWELL(engine->i915));
+
+ engine->emit_flush = gen6_emit_flush_xcs;
+ engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
+ engine->irq_enable = hsw_irq_enable_vecs;
+ engine->irq_disable = hsw_irq_disable_vecs;
}
static unsigned int global_ring_size(void)
@@ -713,6 +1067,58 @@ static unsigned int global_ring_size(void)
return roundup_pow_of_two(EXECLIST_MAX_PORTS * SZ_16K + SZ_4K);
}
+static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
+{
+ struct drm_i915_gem_object *obj;
+ struct i915_vma *vma;
+ int size;
+ int err;
+
+ size = gen7_setup_clear_gpr_bb(engine, NULL /* probe size */);
+ if (size <= 0)
+ return size;
+
+ size = ALIGN(size, PAGE_SIZE);
+ obj = i915_gem_object_create_internal(engine->i915, size);
+ if (IS_ERR(obj))
+ return PTR_ERR(obj);
+
+ vma = i915_vma_instance(obj, engine->gt->vm, NULL);
+ if (IS_ERR(vma)) {
+ err = PTR_ERR(vma);
+ goto err_obj;
+ }
+
+ vma->private = intel_context_create(engine); /* dummy residuals */
+ if (IS_ERR(vma->private)) {
+ err = PTR_ERR(vma->private);
+ goto err_obj;
+ }
+
+ err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
+ if (err)
+ goto err_private;
+
+ err = i915_vma_sync(vma);
+ if (err)
+ goto err_unpin;
+
+ size = gen7_setup_clear_gpr_bb(engine, vma);
+ if (err)
+ goto err_unpin;
+
+ engine->wa_ctx.vma = vma;
+ return 0;
+
+err_unpin:
+ i915_vma_unpin(vma);
+err_private:
+ intel_context_put(vma->private);
+err_obj:
+ i915_gem_object_put(obj);
+ return err;
+}
+
int intel_ring_scheduler_setup(struct intel_engine_cs *engine)
{
struct intel_ring *ring;
@@ -756,13 +1162,23 @@ int intel_ring_scheduler_setup(struct intel_engine_cs *engine)
GEM_BUG_ON(engine->legacy.ring);
engine->legacy.ring = ring;
+ if (IS_HASWELL(engine->i915) && engine->class == RENDER_CLASS) {
+ err = gen7_ctx_switch_bb_init(engine);
+ if (err)
+ goto err_ring_unpin;
+ }
+
engine->flags |= I915_ENGINE_NEEDS_BREADCRUMB_TASKLET;
engine->flags |= I915_ENGINE_SUPPORTS_STATS;
+ if (INTEL_GEN(engine->i915) >= 6)
+ engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
/* Finally, take ownership and responsibility for cleanup! */
engine->release = ring_release;
return 0;
+err_ring_unpin:
+ intel_ring_unpin(ring);
err_ring:
intel_ring_put(ring);
err:
diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
index 3350e7c995bc..5fe254876b94 100644
--- a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c
@@ -263,6 +263,9 @@ static int live_ctx_switch_wa(void *arg)
struct i915_vma *saved_wa;
int err;
+ if (engine->execlists.tasklet.func)
+ continue;
+
if (!intel_engine_can_store_dword(engine))
continue;
--
2.20.1
More information about the Intel-gfx-trybot
mailing list