[PATCH 98/98] virtual-engine
Chris Wilson
chris at chris-wilson.co.uk
Fri Apr 27 11:11:21 UTC 2018
---
drivers/gpu/drm/i915/i915_gem_context.c | 66 ++--
drivers/gpu/drm/i915/i915_gem_context.h | 1 +
drivers/gpu/drm/i915/i915_gem_execbuffer.c | 52 ++-
drivers/gpu/drm/i915/i915_request.c | 3 +-
drivers/gpu/drm/i915/intel_engine_cs.c | 8 +-
drivers/gpu/drm/i915/intel_lrc.c | 380 ++++++++++++++++++++-
drivers/gpu/drm/i915/intel_lrc.h | 7 +
drivers/gpu/drm/i915/intel_ringbuffer.h | 8 +
drivers/gpu/drm/i915/selftests/intel_lrc.c | 198 +++++++++++
include/uapi/drm/i915_drm.h | 20 ++
10 files changed, 677 insertions(+), 66 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index e414873b46f1..435e5ddd8d58 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -92,6 +92,7 @@
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_gt_pm.h"
+#include "intel_lrc.h"
#include "intel_workarounds.h"
#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
@@ -193,6 +194,11 @@ static void i915_gem_context_free(struct i915_gem_context *ctx)
ce->ops->destroy(ce);
}
+ if (ctx->engines) {
+ intel_virtual_engine_put(ctx->engines[0]);
+ kfree(ctx->engines);
+ }
+
if (ctx->timeline)
i915_timeline_put(ctx->timeline);
@@ -907,7 +913,7 @@ static struct intel_engine_cs *lookup_engine(struct drm_i915_private *i915,
unsigned int class,
unsigned int instance)
{
- return NULL;
+ return intel_engine_lookup_user(i915, class, instance);
}
static int set_engines(struct i915_gem_context *ctx,
@@ -917,13 +923,6 @@ static int set_engines(struct i915_gem_context *ctx,
struct intel_engine_cs **engines;
int nengine, n;
- /*
- * For single timeline queues, we allow the user to trim the set
- * of engines, and to ask the kernel to load balance across the set.
- */
- if (!ctx->timeline)
- return -EINVAL;
-
if (args->size == 0)
return -EINVAL;
@@ -936,50 +935,47 @@ static int set_engines(struct i915_gem_context *ctx,
nengine = args->size / sizeof(*uengine);
uengine = u64_to_user_ptr(args->value);
- engines = kmalloc_array(nengine, sizeof(*engines), GFP_KERNEL);
+ engines = kmalloc_array(nengine + 1, sizeof(*engines), GFP_KERNEL);
if (!engines)
return -ENOMEM;
for (n = 0; n < nengine; n++) {
u64 uabi_class_instance;
+ unsigned int class, instance;
if (get_user(uabi_class_instance, &uengine[n])) {
kfree(engines);
return -EFAULT;
}
- engines[n] = lookup_engine(ctx->i915,
- upper_32_bits(uabi_class_instance),
- lower_32_bits(uabi_class_instance));
- if (!engines[n]) {
+ class = upper_32_bits(uabi_class_instance);
+ instance = lower_32_bits(uabi_class_instance);
+
+ engines[n + 1] = lookup_engine(ctx->i915, class, instance);
+ if (!engines[n + 1]) {
kfree(engines);
return -ENOENT;
}
}
- kfree(ctx->engines);
- ctx->engines = engines;
- ctx->nengine = nengine;
+ engines[0] = NULL;
+ if (HAS_EXECLISTS(ctx->i915) && ctx->timeline) {
+ engines[0] = intel_execlists_create_virtual(ctx->i915, ctx,
+ engines + 1,
+ nengine);
+ if (IS_ERR(engines[0])) {
+ int err = PTR_ERR(engines[0]);
+ kfree(engines);
+ return err;
+ }
+ }
- /*
- * Then in execbuf
- *
- * if (ctx->engine) {
- * int id = args->flags & 63;
- *
- * // load balance by default
- *
- * if (id) {
- * if (id >= ctx->nengine)
- * return -EINVAL;
- *
- * engine = ctx->engines[id - 1];
- * } else {
- * engine = load_balance(ctx->engines, ctx->nengine);
- * }
- * }
- *
- */
+ if (ctx->engines) {
+ intel_virtual_engine_put(ctx->engines[0]);
+ kfree(ctx->engines);
+ }
+ ctx->engines = engines;
+ ctx->nengine = nengine + 1;
return 0;
}
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index c6242c28503d..2c4013e362db 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -172,6 +172,7 @@ struct i915_gem_context {
/** engine: per-engine logical HW state */
struct intel_context {
struct i915_gem_context *gem_context;
+ struct intel_engine_cs *active;
struct i915_vma *state;
struct intel_ring *ring;
u32 *lrc_reg_state;
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 44044836c75a..6df43480a6f1 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -1979,13 +1979,23 @@ static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
};
static struct intel_engine_cs *
-eb_select_engine(struct drm_i915_private *dev_priv,
+eb_select_engine(struct i915_execbuffer *eb,
struct drm_file *file,
struct drm_i915_gem_execbuffer2 *args)
{
unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
struct intel_engine_cs *engine;
+ if (eb->ctx->engines) {
+ if (user_ring_id >= eb->ctx->nengine) {
+ DRM_DEBUG("execbuf with unknown ring: %u\n",
+ user_ring_id);
+ return NULL;
+ }
+
+ return eb->ctx->engines[user_ring_id];
+ }
+
if (user_ring_id > I915_USER_RINGS) {
DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
return NULL;
@@ -1998,11 +2008,11 @@ eb_select_engine(struct drm_i915_private *dev_priv,
return NULL;
}
- if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
+ if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(eb->i915)) {
unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
- bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
+ bsd_idx = gen8_dispatch_bsd_engine(eb->i915, file);
} else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
bsd_idx <= I915_EXEC_BSD_RING2) {
bsd_idx >>= I915_EXEC_BSD_SHIFT;
@@ -2013,9 +2023,9 @@ eb_select_engine(struct drm_i915_private *dev_priv,
return NULL;
}
- engine = dev_priv->engine[_VCS(bsd_idx)];
+ engine = eb->i915->engine[_VCS(bsd_idx)];
} else {
- engine = dev_priv->engine[user_ring_map[user_ring_id]];
+ engine = eb->i915->engine[user_ring_map[user_ring_id]];
}
if (!engine) {
@@ -2202,19 +2212,27 @@ i915_gem_do_execbuffer(struct drm_device *dev,
if (args->flags & I915_EXEC_IS_PINNED)
eb.batch_flags |= I915_DISPATCH_PINNED;
- eb.engine = eb_select_engine(eb.i915, file, args);
- if (!eb.engine)
- return -EINVAL;
+ err = eb_select_context(&eb);
+ if (unlikely(err))
+ return err;
+
+ eb.engine = eb_select_engine(&eb, file, args);
+ if (!eb.engine) {
+ err = -EINVAL;
+ goto err_context;
+ }
if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
if (!HAS_RESOURCE_STREAMER(eb.i915)) {
DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
- return -EINVAL;
+ err = -EINVAL;
+ goto err_context;
}
if (eb.engine->id != RCS) {
DRM_DEBUG("RS is not available on %s\n",
eb.engine->name);
- return -EINVAL;
+ err = -EINVAL;
+ goto err_context;
}
eb.batch_flags |= I915_DISPATCH_RS;
@@ -2222,8 +2240,10 @@ i915_gem_do_execbuffer(struct drm_device *dev,
if (args->flags & I915_EXEC_FENCE_IN) {
in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
- if (!in_fence)
- return -EINVAL;
+ if (!in_fence) {
+ err = -EINVAL;
+ goto err_context;
+ }
}
if (args->flags & I915_EXEC_FENCE_OUT) {
@@ -2240,10 +2260,6 @@ i915_gem_do_execbuffer(struct drm_device *dev,
GEM_BUG_ON(!eb.lut_size);
- err = eb_select_context(&eb);
- if (unlikely(err))
- goto err_destroy;
-
/*
* Take a local wakeref for preparing to dispatch the execbuf as
* we expect to access the hardware fairly frequently in the
@@ -2404,14 +2420,14 @@ i915_gem_do_execbuffer(struct drm_device *dev,
mutex_unlock(&dev->struct_mutex);
err_rpm:
intel_runtime_pm_put(eb.i915);
- i915_gem_context_put(eb.ctx);
-err_destroy:
eb_destroy(&eb);
err_out_fence:
if (out_fence_fd != -1)
put_unused_fd(out_fence_fd);
err_in_fence:
dma_fence_put(in_fence);
+err_context:
+ i915_gem_context_put(eb.ctx);
return err;
}
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 3be5008d9e64..13feb3020f6a 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -353,6 +353,7 @@ static void __retire_engine_request(struct intel_engine_cs *engine,
intel_engine_get_seqno(engine));
GEM_BUG_ON(!i915_request_completed(rq));
+ GEM_BUG_ON(intel_engine_is_virtual(engine));
local_irq_disable();
@@ -1097,7 +1098,7 @@ void __i915_request_add(struct i915_request *request, bool flush_caches)
prev = i915_gem_active_raw(&timeline->last_request,
&request->i915->drm.struct_mutex);
if (prev && !i915_request_completed(prev)) {
- if (prev->engine == engine)
+ if (prev->engine == engine && !intel_engine_is_virtual(engine))
i915_sw_fence_await_sw_fence(&request->submit,
&prev->submit,
&request->submitq);
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index f7625f67844c..866ba8992760 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -759,9 +759,11 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
if (engine->default_state)
i915_gem_object_put(engine->default_state);
- if (i915->preempt_context)
- __intel_context_unpin(i915->preempt_context, engine);
- __intel_context_unpin(i915->kernel_context, engine);
+ if (!intel_engine_is_virtual(engine)) { /* XXX */
+ if (i915->preempt_context)
+ __intel_context_unpin(i915->preempt_context, engine);
+ __intel_context_unpin(i915->kernel_context, engine);
+ }
i915_timeline_fini(&engine->timeline);
}
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 1a2865a5b0e9..ee3f0ae40b6c 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -164,6 +164,26 @@
#define WA_TAIL_DWORDS 2
#define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
+struct virtual_engine {
+ struct intel_engine_cs base;
+
+ struct intel_context context;
+ struct kref kref;
+
+ struct intel_engine_cs *bound;
+
+ struct i915_request *request;
+ struct rb_node node[I915_NUM_ENGINES];
+
+ unsigned int count;
+ struct intel_engine_cs *siblings[0];
+};
+
+static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine)
+{
+ return container_of(engine, struct virtual_engine, base);
+}
+
static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
struct intel_engine_cs *engine,
struct intel_context *ce);
@@ -476,8 +496,11 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
if (rq) {
GEM_BUG_ON(!rq->hw_context->pin_count);
GEM_BUG_ON(count > !n);
- if (!count++)
+ if (!count++) {
+ GEM_BUG_ON(rq->hw_context->active);
execlists_context_schedule_in(rq);
+ rq->hw_context->active = engine;
+ }
port_set(&port[n], port_pack(rq, count));
desc = execlists_update_context(rq);
GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
@@ -676,6 +699,47 @@ static void complete_preempt_context(struct intel_engine_execlists *execlists)
execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
}
+static void virtual_update_register_offsets(u32 *regs,
+ struct intel_engine_cs *engine)
+{
+ u32 base = engine->mmio_base;
+
+ regs[CTX_CONTEXT_CONTROL] = i915_mmio_reg_offset(RING_CONTEXT_CONTROL(engine));
+ regs[CTX_RING_HEAD] = i915_mmio_reg_offset(RING_HEAD(base));
+ regs[CTX_RING_TAIL] = i915_mmio_reg_offset(RING_TAIL(base));
+ regs[CTX_RING_BUFFER_START] = i915_mmio_reg_offset(RING_START(base));
+ regs[CTX_RING_BUFFER_CONTROL] = i915_mmio_reg_offset(RING_CTL(base));
+
+ regs[CTX_BB_HEAD_U] = i915_mmio_reg_offset(RING_BBADDR_UDW(base));
+ regs[CTX_BB_HEAD_L] = i915_mmio_reg_offset(RING_BBADDR(base));
+ regs[CTX_BB_STATE] = i915_mmio_reg_offset(RING_BBSTATE(base));
+ regs[CTX_SECOND_BB_HEAD_U] = i915_mmio_reg_offset(RING_SBBADDR_UDW(base));
+ regs[CTX_SECOND_BB_HEAD_L] = i915_mmio_reg_offset(RING_SBBADDR(base));
+ regs[CTX_SECOND_BB_STATE] = i915_mmio_reg_offset(RING_SBBSTATE(base));
+
+ regs[CTX_CTX_TIMESTAMP] = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(base));
+ regs[CTX_PDP3_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, 3));
+ regs[CTX_PDP3_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, 3));
+ regs[CTX_PDP2_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, 2));
+ regs[CTX_PDP2_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, 2));
+ regs[CTX_PDP1_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, 1));
+ regs[CTX_PDP1_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, 1));
+ regs[CTX_PDP0_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, 0));
+ regs[CTX_PDP0_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, 0));
+
+ if (engine->class == RENDER_CLASS) {
+ regs[CTX_RCS_INDIRECT_CTX] =
+ i915_mmio_reg_offset(RING_INDIRECT_CTX(base));
+ regs[CTX_RCS_INDIRECT_CTX_OFFSET] =
+ i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(base));
+ regs[CTX_BB_PER_CTX_PTR] =
+ i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(base));
+
+ regs[CTX_R_PWR_CLK_STATE] =
+ i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE);
+ }
+}
+
static void execlists_dequeue(struct intel_engine_cs *engine)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -686,6 +750,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
struct rb_node *rb;
unsigned long flags;
bool submit = false;
+ int prio;
/* Hardware submission is through 2 ports. Conceptually each port
* has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
@@ -710,6 +775,29 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
spin_lock_irqsave(&engine->timeline.lock, flags);
+ prio = execlists->queue_priority;
+ for (rb = rb_first_cached(&execlists->virtual); rb; ) {
+ struct virtual_engine *ve =
+ rb_entry(rb, typeof(*ve), node[engine->id]);
+ struct intel_engine_cs *active;
+
+ if (!ve->request) {
+ rb_erase_cached(rb, &execlists->virtual);
+ RB_CLEAR_NODE(rb);
+ rb = rb_first_cached(&execlists->virtual);
+ continue;
+ }
+
+ active = READ_ONCE(ve->context.active);
+ if (active && active != engine) {
+ rb = rb_next(rb);
+ continue;
+ }
+
+ prio = max(prio, rq_prio(ve->request));
+ break;
+ }
+
if (last) {
/*
* Don't resubmit or switch until all outstanding
@@ -733,7 +821,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
goto unlock;
- if (need_preempt(engine, last, execlists->queue_priority)) {
+ if (need_preempt(engine, last, prio)) {
inject_preempt_context(engine);
goto unlock;
}
@@ -773,6 +861,44 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
last->tail = last->wa_tail;
}
+ if (rb) {
+ struct virtual_engine *ve =
+ rb_entry(rb, typeof(*ve), node[engine->id]);
+
+ /* XXX virtual is always taking precedence */
+ spin_lock(&ve->base.timeline.lock);
+ if (ve->request && rq_prio(ve->request) >= prio) {
+ struct i915_request *rq = ve->request;
+
+ if (last && !can_merge_rq(rq, last)) {
+ GEM_BUG_ON(port == last_port);
+ spin_unlock(&ve->base.timeline.lock);
+ goto unlock;
+ }
+
+ GEM_BUG_ON(rq->engine != &ve->base);
+ GEM_BUG_ON(rq->hw_context != &ve->context);
+ rq->engine = engine;
+
+ if (engine != ve->bound) {
+ u32 *regs = ve->context.lrc_reg_state;
+
+ virtual_update_register_offsets(regs, engine);
+ ve->bound = engine;
+ }
+
+ __i915_request_submit(rq);
+ trace_i915_request_in(rq, port_index(port, execlists));
+ submit = true;
+ last = rq;
+
+ ve->request = NULL;
+ rb_erase_cached(rb, &execlists->virtual);
+ RB_CLEAR_NODE(rb);
+ }
+ spin_unlock(&ve->base.timeline.lock);
+ }
+
while ((rb = rb_first_cached(&execlists->queue))) {
struct i915_priolist *p = to_priolist(rb);
struct i915_request *rq, *rn;
@@ -896,6 +1022,7 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
intel_engine_get_seqno(rq->engine));
GEM_BUG_ON(!execlists->active);
+ rq->hw_context->active = NULL;
intel_engine_context_out(rq->engine);
execlists_context_status_change(rq,
@@ -1198,6 +1325,7 @@ static void process_csb(struct intel_engine_cs *engine)
*/
GEM_BUG_ON(!i915_request_completed(rq));
+ rq->hw_context->active = NULL;
execlists_context_schedule_out(rq);
trace_i915_request_out(rq);
@@ -1320,21 +1448,25 @@ static void submit_queue(struct intel_engine_cs *engine,
__submit_queue(engine, prio, timeout);
}
-static void execlists_submit_request(struct i915_request *request)
+static void __execlists_submit_request(struct intel_engine_cs *engine,
+ struct i915_request *request)
{
- struct intel_engine_cs *engine = request->engine;
- unsigned long flags;
-
- /* Will be called from irq-context when using foreign fences. */
- spin_lock_irqsave(&engine->timeline.lock, flags);
-
queue_request(engine, &request->sched, rq_prio(request));
submit_queue(engine,
rq_prio(request), request->gem_context->preempt_timeout);
GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
GEM_BUG_ON(list_empty(&request->sched.link));
+}
+static void execlists_submit_request(struct i915_request *request)
+{
+ struct intel_engine_cs *engine = request->engine;
+ unsigned long flags;
+
+ /* Will be called from irq-context when using foreign fences. */
+ spin_lock_irqsave(&engine->timeline.lock, flags);
+ __execlists_submit_request(engine, request);
spin_unlock_irqrestore(&engine->timeline.lock, flags);
}
@@ -2988,6 +3120,236 @@ void intel_lr_context_resume(struct drm_i915_private *i915)
}
}
+static void virtual_engine_free(struct kref *kref)
+{
+ struct virtual_engine *ve = container_of(kref, typeof(*ve), kref);
+ struct intel_context *ce = &ve->context;
+ unsigned int n;
+
+ GEM_BUG_ON(ve->request);
+ tasklet_kill(&ve->base.execlists.tasklet);
+
+ for (n = 0; n < ve->count; n++) {
+ struct intel_engine_cs *sibling = ve->siblings[n];
+ struct rb_node *node = &ve->node[sibling->id];
+
+ spin_lock(&sibling->timeline.lock);
+
+ if (!RB_EMPTY_NODE(node))
+ rb_erase_cached(node, &sibling->execlists.virtual);
+
+ spin_unlock(&sibling->timeline.lock);
+
+ tasklet_kill(&ve->siblings[n]->execlists.tasklet);
+ }
+
+ if (ce->state)
+ execlists_context_destroy(ce);
+
+ i915_timeline_fini(&ve->base.timeline);
+ //intel_engine_cleanup_common(&ve->base);
+ kfree(ve);
+}
+
+static void virtual_context_unpin(struct intel_context *ce)
+{
+ struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
+
+ lockdep_assert_held(&ce->gem_context->i915->drm.struct_mutex);
+ GEM_BUG_ON(ce->pin_count == 0);
+
+ if (--ce->pin_count)
+ return;
+
+ __execlists_context_unpin(ce);
+ kref_put(&ve->kref, virtual_engine_free);
+}
+
+static const struct intel_context_ops virtual_context_ops = {
+ .unpin = virtual_context_unpin,
+};
+
+static struct intel_context *
+virtual_context_pin(struct intel_engine_cs *engine,
+ struct i915_gem_context *ctx)
+{
+ struct virtual_engine *ve = to_virtual_engine(engine);
+ struct intel_context *ce = &ve->context;
+
+ lockdep_assert_held(&ctx->i915->drm.struct_mutex);
+
+ if (likely(ce->pin_count++))
+ return ce;
+ GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
+
+ kref_get(&ve->kref);
+ ce->ops = &virtual_context_ops;
+
+ return __execlists_context_pin(engine, ctx, ce);
+}
+
+static void virtual_submission_tasklet(unsigned long data)
+{
+ struct virtual_engine * const ve = (struct virtual_engine *)data;
+ unsigned int n;
+ int prio;
+
+ local_irq_disable();
+
+ prio = I915_PRIORITY_INVALID;
+ spin_lock(&ve->base.timeline.lock);
+ if (ve->request)
+ prio = rq_prio(ve->request);
+ spin_unlock(&ve->base.timeline.lock);
+ if (prio == I915_PRIORITY_INVALID)
+ goto out;
+
+ for (n = 0; n < ve->count; n++) {
+ struct intel_engine_cs *sibling = ve->siblings[n];
+ struct rb_node *node = &ve->node[sibling->id];
+ struct rb_node **parent, *rb;
+ bool first = true;
+
+ spin_lock(&sibling->timeline.lock);
+
+ if (!RB_EMPTY_NODE(node))
+ rb_erase_cached(node, &sibling->execlists.virtual);
+
+ rb = NULL;
+ parent = &sibling->execlists.virtual.rb_root.rb_node;
+ while (*parent) {
+ struct virtual_engine *other;
+
+ rb = *parent;
+ other = rb_entry(rb, typeof(*other), node[sibling->id]);
+ if (other->request && prio > rq_prio(other->request)) {
+ parent = &rb->rb_left;
+ } else {
+ parent = &rb->rb_right;
+ first = false;
+ }
+ }
+
+ rb_link_node(node, rb, parent);
+ rb_insert_color_cached(node,
+ &sibling->execlists.virtual,
+ first);
+ if (first && prio > sibling->execlists.queue_priority)
+ tasklet_hi_schedule(&sibling->execlists.tasklet);
+
+ spin_unlock(&sibling->timeline.lock);
+ }
+
+out:
+ local_irq_enable();
+}
+
+static void virtual_submit_request(struct i915_request *request)
+{
+ struct virtual_engine *ve = to_virtual_engine(request->engine);
+ unsigned long flags;
+
+ spin_lock_irqsave(&ve->base.timeline.lock, flags);
+
+ GEM_BUG_ON(ve->request);
+ ve->request = request;
+
+ spin_unlock_irqrestore(&ve->base.timeline.lock, flags);
+
+ tasklet_schedule(&ve->base.execlists.tasklet);
+}
+
+struct intel_engine_cs *
+intel_execlists_create_virtual(struct drm_i915_private *i915,
+ struct i915_gem_context *ctx,
+ struct intel_engine_cs **siblings,
+ unsigned int count)
+{
+ struct virtual_engine *ve;
+ unsigned int n;
+ int err;
+
+ if (!count)
+ return ERR_PTR(-EINVAL);
+
+ ve = kzalloc(sizeof(*ve) + count * sizeof(*ve->siblings), GFP_KERNEL);
+ if (!ve)
+ return ERR_PTR(-ENOMEM);
+
+ kref_init(&ve->kref);
+ ve->base.i915 = i915;
+ ve->base.id = -1;
+ ve->base.class = OTHER_CLASS;
+ ve->base.flags = I915_ENGINE_IS_VIRTUAL;
+
+ snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
+
+ intel_engine_setup_common(&ve->base);
+ INIT_LIST_HEAD(&ve->base.breadcrumbs.signals); /* XXX */
+
+ ve->context.gem_context = ctx;
+
+ ve->base.context_pin = virtual_context_pin;
+ ve->base.request_alloc = execlists_request_alloc;
+
+ ve->base.schedule = execlists_schedule;
+ ve->base.submit_request = virtual_submit_request;
+
+ tasklet_init(&ve->base.execlists.tasklet,
+ virtual_submission_tasklet,
+ (unsigned long)ve);
+
+ err = intel_engine_create_scratch(&ve->base, PAGE_SIZE);
+ if (err)
+ goto err_put;
+
+ for (n = 0; n < ARRAY_SIZE(ve->node); n++)
+ RB_CLEAR_NODE(&ve->node[n]);
+
+ ve->count = count;
+ for (n = 0; n < count; n++) {
+ struct intel_engine_cs *sibling = siblings[n];
+
+ ve->siblings[n] = sibling;
+
+ if (ve->base.class != OTHER_CLASS) {
+ if (ve->base.class != sibling->class) {
+ err = -EINVAL;
+ goto err_put;
+ }
+ continue;
+ }
+
+ ve->base.class = sibling->class;
+ snprintf(ve->base.name, sizeof(ve->base.name),
+ "v%dx%d", ve->base.class, count);
+ ve->base.context_size = sibling->context_size;
+
+ /* XXX single default state per class? */
+ ve->base.default_state =
+ i915_gem_object_get(sibling->default_state);
+
+ ve->base.emit_bb_start = sibling->emit_bb_start;
+ ve->base.emit_flush = sibling->emit_flush;
+ ve->base.emit_breadcrumb = sibling->emit_breadcrumb;
+ ve->base.emit_breadcrumb_sz = sibling->emit_breadcrumb_sz;
+ }
+
+ return &ve->base;
+
+err_put:
+ virtual_engine_free(&ve->kref);
+ return ERR_PTR(err);
+}
+
+void intel_virtual_engine_put(struct intel_engine_cs *engine)
+{
+ if (!engine)
+ return;
+
+ kref_put(&to_virtual_engine(engine)->kref, virtual_engine_free);
+}
+
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/intel_lrc.c"
#endif
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 1593194e930c..91b1954f744e 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -104,4 +104,11 @@ struct i915_gem_context;
void intel_lr_context_resume(struct drm_i915_private *dev_priv);
+struct intel_engine_cs *
+intel_execlists_create_virtual(struct drm_i915_private *i915,
+ struct i915_gem_context *ctx,
+ struct intel_engine_cs **siblings,
+ unsigned int count);
+void intel_virtual_engine_put(struct intel_engine_cs *engine);
+
#endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 4c6a435c5a5f..e85907954aaf 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -291,6 +291,7 @@ struct intel_engine_execlists {
* @queue: queue of requests, in priority lists
*/
struct rb_root_cached queue;
+ struct rb_root_cached virtual;
/**
* @fw_domains: forcewake domains for irq tasklet
@@ -572,6 +573,7 @@ struct intel_engine_cs {
#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0)
#define I915_ENGINE_SUPPORTS_STATS BIT(1)
#define I915_ENGINE_HAS_PREEMPTION BIT(2)
+#define I915_ENGINE_IS_VIRTUAL BIT(3)
unsigned int flags;
/*
@@ -654,6 +656,12 @@ static inline bool __execlists_need_preempt(int prio, int last)
return prio > max(0, last);
}
+static inline bool
+intel_engine_is_virtual(const struct intel_engine_cs *engine)
+{
+ return engine->flags & I915_ENGINE_IS_VIRTUAL;
+}
+
static inline void
execlists_set_active(struct intel_engine_execlists *execlists,
unsigned int bit)
diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c
index 71cadef690f7..bb61c6a2ae47 100644
--- a/drivers/gpu/drm/i915/selftests/intel_lrc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c
@@ -4,6 +4,8 @@
* Copyright © 2018 Intel Corporation
*/
+#include <linux/prime_numbers.h>
+
#include "../i915_selftest.h"
#include "mock_context.h"
@@ -885,6 +887,201 @@ static int live_context_preempt_timeout(void *arg)
return err;
}
+struct live_test {
+ struct drm_i915_private *i915;
+ const char *func;
+ const char *name;
+
+ unsigned int reset_count;
+ bool wedge;
+};
+
+static int begin_live_test(struct live_test *t,
+ struct drm_i915_private *i915,
+ const char *func,
+ const char *name)
+{
+ struct wedge_me wedge;
+ int err;
+
+ t->i915 = i915;
+ t->func = func;
+ t->name = name;
+
+ wedge_on_timeout(&wedge, i915, 10*HZ) {
+ err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+ if (err) {
+ pr_err("%s(%s): failed to idle before, with err=%d!",
+ func, name, err);
+ return err;
+ }
+ }
+
+ if (i915_terminally_wedged(&i915->gpu_error))
+ return -EIO;
+
+ i915->gpu_error.missed_irq_rings = 0;
+ t->reset_count = i915_reset_count(&i915->gpu_error);
+
+ return 0;
+}
+
+static int end_live_test(struct live_test *t)
+{
+ struct drm_i915_private *i915 = t->i915;
+ struct wedge_me wedge;
+
+ i915_retire_requests(i915);
+
+ wedge_on_timeout(&wedge, i915, 10*HZ) {
+ if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)) {
+ pr_err("%s(%s): failed to idle\n", t->func, t->name);
+ return -EIO;
+ }
+ }
+
+ if (i915_terminally_wedged(&i915->gpu_error)) {
+ pr_err("%s(%s): *** wedged ****\n", t->func, t->name);
+ return -EIO;
+ }
+
+ if (t->reset_count != i915_reset_count(&i915->gpu_error)) {
+ pr_err("%s(%s): GPU was reset %d times!\n",
+ t->func, t->name,
+ i915_reset_count(&i915->gpu_error) - t->reset_count);
+ return -EIO;
+ }
+
+ if (i915->gpu_error.missed_irq_rings) {
+ pr_err("%s(%s): Missed interrupts on engines %lx\n",
+ t->func, t->name, i915->gpu_error.missed_irq_rings);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static int nop_virtual_engine(struct drm_i915_private *i915,
+ struct intel_engine_cs **siblings,
+ unsigned int nsibling,
+ unsigned int nctx)
+{
+ IGT_TIMEOUT(end_time);
+ struct i915_request *request[16];
+ struct i915_gem_context *ctx[16];
+ struct intel_engine_cs *ve[16];
+ unsigned long n, prime, nc;
+ ktime_t times[2] = {};
+ struct live_test t;
+ int err;
+
+ GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx));
+
+ for (n = 0; n < nctx; n++) {
+ ctx[n] = kernel_context(i915);
+ if (!ctx[n])
+ return -ENOMEM;
+
+ ve[n] = intel_execlists_create_virtual(i915, ctx[n],
+ siblings, nsibling);
+ if (IS_ERR(ve[n]))
+ return PTR_ERR(ve[n]);
+ }
+
+ err = begin_live_test(&t, i915, __func__, ve[0]->name);
+ if (err)
+ goto out;
+
+ for_each_prime_number_from(prime, 1, 8192) {
+ times[1] = ktime_get_raw();
+
+ for (nc = 0; nc < nctx; nc++) {
+ for (n = 0; n < prime; n++) {
+ request[nc] =
+ i915_request_alloc(ve[nc], ctx[nc]);
+ if (IS_ERR(request[nc])) {
+ err = PTR_ERR(request[nc]);
+ goto out;
+ }
+
+ i915_request_add(request[nc]);
+ }
+ }
+
+ for (nc = 0; nc < nctx; nc++)
+ i915_request_wait(request[nc],
+ I915_WAIT_LOCKED,
+ MAX_SCHEDULE_TIMEOUT);
+
+ times[1] = ktime_sub(ktime_get_raw(), times[1]);
+ if (prime == 1)
+ times[0] = times[1];
+
+ if (__igt_timeout(end_time, NULL))
+ break;
+ }
+
+ err = end_live_test(&t);
+ if (err)
+ goto out;
+
+ pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
+ nctx, ve[0]->name, ktime_to_ns(times[0]),
+ prime, div64_u64(ktime_to_ns(times[1]), prime));
+
+out:
+ for (nc = 0; nc < nctx; nc++) {
+ intel_virtual_engine_put(ve[nc]);
+ kernel_context_close(ctx[nc]);
+ }
+ return err;
+}
+
+static int live_virtual_engine(void *arg)
+{
+ struct drm_i915_private *i915 = arg;
+ struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+ unsigned int class, inst;
+ int err = -ENODEV;
+
+ mutex_lock(&i915->drm.struct_mutex);
+
+ for_each_engine(engine, i915, id) {
+ err = nop_virtual_engine(i915, &engine, 1, 1);
+ if (err) {
+ pr_err("Failed to wrap engine %s: err=%d\n",
+ engine->name, err);
+ goto out_unlock;
+ }
+ }
+
+ for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
+ int nsibling, n;
+
+ nsibling = 0;
+ for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
+ if (!i915->engine_class[class][inst])
+ break;
+
+ siblings[nsibling++] = i915->engine_class[class][inst];
+ }
+ if (nsibling < 2)
+ continue;
+
+ for (n = 1; n <= nsibling + 1; n++) {
+ err = nop_virtual_engine(i915, siblings, nsibling, n);
+ if (err)
+ goto out_unlock;
+ }
+ }
+
+out_unlock:
+ mutex_unlock(&i915->drm.struct_mutex);
+ return err;
+}
+
int intel_execlists_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
@@ -895,6 +1092,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915)
SUBTEST(live_preempt_reset),
SUBTEST(live_late_preempt_timeout),
SUBTEST(live_context_preempt_timeout),
+ SUBTEST(live_virtual_engine),
};
return i915_subtests(tests, i915);
}
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 90c0ce13ab75..a0dcdbeff2ae 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -1543,11 +1543,31 @@ struct drm_i915_gem_context_param {
#define I915_CONTEXT_MAX_FREQUENCY(x) ((x) >> 32)
#define I915_CONTEXT_SET_FREQUENCY(min, max) ((__u64)(max) << 32 | (min))
+/*
+ * I915_CONTEXT_PARAM_ENGINES:
+ *
+ * Bind this context to operate on this subset of available engines. Henceforth,
+ * the I915_EXEC_RING selector for DRM_IOCTL_I915_GEM_EXECBUFFER2 operates as
+ * an index into this array of engines; I915_EXEC_DEFAULT selecting engine[0]
+ * and upwards. The array created is offset by 1, such that by default
+ * I915_EXEC_DEFAULT is left empty, to be filled in as directed. In particular,
+ * it can be used as a load-balancing engine.
+ *
+ * See struct i915_context_param_engines.
+ */
#define I915_CONTEXT_PARAM_ENGINES 0x9
__u64 value;
};
+struct i915_context_param_engines {
+ __u64 flags;
+#define I915_CONTEXT_PARAM_ENGINES_CREATE_VIRTUAL 0x1
+ __u64 mbz;
+
+ __u64 class_instance[0]; /* class << 32 | instance */
+};
+
enum drm_i915_oa_format {
I915_OA_FORMAT_A13 = 1, /* HSW only */
I915_OA_FORMAT_A29, /* HSW only */
--
2.17.0
More information about the Intel-gfx-trybot
mailing list