Number of available GuC contexts ids might be limited. Stop referring in code to macro and use variable instead.

Signed-off-by: Michal Wajdeczko michal.wajdeczko@intel.com Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/uc/intel_guc.h | 2 ++ .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index a9547069ee7e..1d7cb118e70f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -53,6 +53,8 @@ struct intel_guc { */ spinlock_t contexts_lock; struct ida guc_ids; + u32 num_guc_ids; + u32 max_guc_ids; struct list_head guc_id_list;

bool submission_supported; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 89ff0e4b4bc7..abfccec7d062 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -275,7 +275,7 @@ static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, u32 index) { struct guc_lrc_desc *base = guc->lrc_desc_pool_vaddr;

- GEM_BUG_ON(index >= GUC_MAX_LRC_DESCRIPTORS); + GEM_BUG_ON(index >= guc->max_guc_ids);

return &base[index]; } @@ -284,7 +284,7 @@ static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id) { struct intel_context *ce = xa_load(&guc->context_lookup, id);

- GEM_BUG_ON(id >= GUC_MAX_LRC_DESCRIPTORS); + GEM_BUG_ON(id >= guc->max_guc_ids);

return ce; } @@ -294,8 +294,7 @@ static int guc_lrc_desc_pool_create(struct intel_guc *guc) u32 size; int ret;

- size = PAGE_ALIGN(sizeof(struct guc_lrc_desc) * - GUC_MAX_LRC_DESCRIPTORS); + size = PAGE_ALIGN(sizeof(struct guc_lrc_desc) * guc->max_guc_ids); ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool, (void **)&guc->lrc_desc_pool_vaddr); if (ret) @@ -1070,7 +1069,7 @@ static void guc_submit_request(struct i915_request *rq) static int new_guc_id(struct intel_guc *guc) { return ida_simple_get(&guc->guc_ids, 0, - GUC_MAX_LRC_DESCRIPTORS, GFP_KERNEL | + guc->num_guc_ids, GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); }

@@ -2562,6 +2561,8 @@ static bool __guc_submission_selected(struct intel_guc *guc)

void intel_guc_submission_init_early(struct intel_guc *guc) { + guc->max_guc_ids = GUC_MAX_LRC_DESCRIPTORS; + guc->num_guc_ids = GUC_MAX_LRC_DESCRIPTORS; guc->submission_supported = __guc_submission_supported(guc); guc->submission_selected = __guc_submission_selected(guc); } @@ -2571,7 +2572,7 @@ g2h_context_lookup(struct intel_guc *guc, u32 desc_idx) { struct intel_context *ce;

- if (unlikely(desc_idx >= GUC_MAX_LRC_DESCRIPTORS)) { + if (unlikely(desc_idx >= guc->max_guc_ids)) { drm_err(&guc_to_gt(guc)->i915->drm, "Invalid desc_idx %u", desc_idx); return NULL; @@ -2874,6 +2875,8 @@ void intel_guc_submission_print_info(struct intel_guc *guc,

drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n", atomic_read(&guc->outstanding_submission_g2h)); + drm_printf(p, "GuC Number GuC IDs: %u\n", guc->num_guc_ids); + drm_printf(p, "GuC Max GuC IDs: %u\n", guc->max_guc_ids); drm_printf(p, "GuC tasklet count: %u\n\n", atomic_read(&sched_engine->tasklet.count));

@@ -2913,7 +2916,6 @@ void intel_guc_submission_print_context_info(struct intel_guc *guc, { struct intel_context *ce; unsigned long index; - xa_for_each(&guc->context_lookup, index, ce) { drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id); drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);

Rather than returning -EAGAIN to the user when no guc_ids are available, implement a fair sharing algorithm in the kernel which blocks submissons until guc_ids become available. Submissions are released one at a time, based on priority, until the guc_id pressure is released to ensure fair sharing of the guc_ids. Once the pressure is fully released, the normal guc_id allocation (at request creation time in guc_request_alloc) can resume as this allocation path should be significantly faster and a fair sharing algorithm isn't needed when guc_ids are plentifully.

The fair sharing algorithm is implemented by forcing all submissions to the tasklet which serializes submissions, dequeuing one at a time.

If the submission doesn't have a guc_id and new guc_id can't be found, two lists are searched, one list with contexts that are not pinned but still registered with the guc (searched first) and another list with contexts that are pinned but do not have any submissions actively in inflight (scheduling enabled + registered, searched second). If no guc_ids can be found we kick a workqueue which will retire requests hopefully freeing a guc_id. The workqueue + tasklet ping / pong back and forth until a guc_id can be found.

Once a guc_id is found, we may have to disable context scheduling depending on which list the context is stolen from. When we disable scheduling, we block the tasklet from executing until the completion G2H returns. The disable scheduling must be issued from the workqueue because of the locking structure. When we deregister a context, we also do the same thing (waiting on the G2H) but we can safely issue the deregister H2G from the tasklet.

Once all the G2H have returned we can trigger a submission on the context.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_context_types.h | 3 + drivers/gpu/drm/i915/gt/uc/intel_guc.h | 26 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 805 ++++++++++++++++-- drivers/gpu/drm/i915/i915_request.h | 6 + 4 files changed, 754 insertions(+), 86 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index e54351a170e2..8ed964ef967b 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -185,6 +185,9 @@ struct intel_context { /* GuC LRC descriptor reference count */ atomic_t guc_id_ref;

+ /* Number of rq submitted without a guc_id */ + u16 guc_num_rq_submit_no_id; + /* * GuC ID link - in list when unpinned but guc_id still valid in GuC */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 1d7cb118e70f..e76579396efd 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -33,7 +33,28 @@ struct intel_guc {

/* Global engine used to submit requests to GuC */ struct i915_sched_engine *sched_engine; - struct i915_request *stalled_request; + + /* Global state related to submission tasklet */ + struct i915_request *stalled_rq; + struct intel_context *stalled_context; + struct work_struct retire_worker; + unsigned long flags; + int total_num_rq_with_no_guc_id; + + /* + * Submisson stall reason. See intel_guc_submission.c for detailed + * description. + */ + enum { + STALL_NONE, + STALL_GUC_ID_WORKQUEUE, + STALL_GUC_ID_TASKLET, + STALL_SCHED_DISABLE, + STALL_REGISTER_CONTEXT, + STALL_DEREGISTER_CONTEXT, + STALL_MOVE_LRC_TAIL, + STALL_ADD_REQUEST, + } submission_stall_reason;

/* intel_guc_recv interrupt related state */ spinlock_t irq_lock; @@ -55,7 +76,8 @@ struct intel_guc { struct ida guc_ids; u32 num_guc_ids; u32 max_guc_ids; - struct list_head guc_id_list; + struct list_head guc_id_list_no_ref; + struct list_head guc_id_list_unpinned;

bool submission_supported; bool submission_selected; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 3b555c05c01c..f42a707f60ca 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -59,6 +59,25 @@ * ELSP context descriptor dword into Work Item. * See guc_add_request() * + * GuC flow control state machine: + * The tasklet, workqueue (retire_worker), and the G2H handlers together more or + * less form a state machine which is used to submit requests + flow control + * requests, while waiting on resources / actions, if necessary. The enum, + * submission_stall_reason, controls the handoff of stalls between these + * entities with stalled_rq & stalled_context being the arguments. Each state + * described below. + * + * STALL_NONE No stall condition + * STALL_GUC_ID_WORKQUEUE Workqueue will try to free guc_ids + * STALL_GUC_ID_TASKLET Tasklet will try to find guc_id + * STALL_SCHED_DISABLE Workqueue will issue context schedule disable + * H2G + * STALL_REGISTER_CONTEXT Tasklet needs to register context + * STALL_DEREGISTER_CONTEXT G2H handler is waiting for context deregister, + * will register context upon receipt of G2H + * STALL_MOVE_LRC_TAIL Tasklet will try to move LRC tail + * STALL_ADD_REQUEST Tasklet will try to add the request (submit + * context) */

/* GuC Virtual Engine */ @@ -72,6 +91,83 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count);

#define GUC_REQUEST_SIZE 64 /* bytes */

+/* + * Global GuC flags helper functions + */ +enum { + GUC_STATE_TASKLET_BLOCKED, + GUC_STATE_GUC_IDS_EXHAUSTED, +}; + +static bool tasklet_blocked(struct intel_guc *guc) +{ + return test_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); +} + +static void set_tasklet_blocked(struct intel_guc *guc) +{ + lockdep_assert_held(&guc->sched_engine->lock); + set_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); +} + +static void __clr_tasklet_blocked(struct intel_guc *guc) +{ + lockdep_assert_held(&guc->sched_engine->lock); + clear_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); +} + +static void clr_tasklet_blocked(struct intel_guc *guc) +{ + unsigned long flags; + + spin_lock_irqsave(&guc->sched_engine->lock, flags); + __clr_tasklet_blocked(guc); + spin_unlock_irqrestore(&guc->sched_engine->lock, flags); +} + +static bool guc_ids_exhausted(struct intel_guc *guc) +{ + return test_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); +} + +static bool test_and_update_guc_ids_exhausted(struct intel_guc *guc) +{ + unsigned long flags; + bool ret = false; + + /* + * Strict ordering on checking if guc_ids are exhausted isn't required, + * so let's avoid grabbing the submission lock if possible. + */ + if (guc_ids_exhausted(guc)) { + spin_lock_irqsave(&guc->sched_engine->lock, flags); + ret = guc_ids_exhausted(guc); + if (ret) + ++guc->total_num_rq_with_no_guc_id; + spin_unlock_irqrestore(&guc->sched_engine->lock, flags); + } + + return ret; +} + +static void set_and_update_guc_ids_exhausted(struct intel_guc *guc) +{ + unsigned long flags; + + spin_lock_irqsave(&guc->sched_engine->lock, flags); + ++guc->total_num_rq_with_no_guc_id; + set_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); + spin_unlock_irqrestore(&guc->sched_engine->lock, flags); +} + +static void clr_guc_ids_exhausted(struct intel_guc *guc) +{ + lockdep_assert_held(&guc->sched_engine->lock); + GEM_BUG_ON(guc->total_num_rq_with_no_guc_id); + + clear_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); +} + /* * Below is a set of functions which control the GuC scheduling state which do * not require a lock as all state transitions are mutually exclusive. i.e. It @@ -82,6 +178,9 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count); #define SCHED_STATE_NO_LOCK_ENABLED BIT(0) #define SCHED_STATE_NO_LOCK_PENDING_ENABLE BIT(1) #define SCHED_STATE_NO_LOCK_REGISTERED BIT(2) +#define SCHED_STATE_NO_LOCK_BLOCK_TASKLET BIT(3) +#define SCHED_STATE_NO_LOCK_GUC_ID_STOLEN BIT(4) +#define SCHED_STATE_NO_LOCK_NEEDS_REGISTER BIT(5) static inline bool context_enabled(struct intel_context *ce) { return (atomic_read(&ce->guc_sched_state_no_lock) & @@ -135,6 +234,60 @@ static inline void clr_context_registered(struct intel_context *ce) &ce->guc_sched_state_no_lock); }

+static inline bool context_block_tasklet(struct intel_context *ce) +{ + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_BLOCK_TASKLET); +} + +static inline void set_context_block_tasklet(struct intel_context *ce) +{ + atomic_or(SCHED_STATE_NO_LOCK_BLOCK_TASKLET, + &ce->guc_sched_state_no_lock); +} + +static inline void clr_context_block_tasklet(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_BLOCK_TASKLET, + &ce->guc_sched_state_no_lock); +} + +static inline bool context_guc_id_stolen(struct intel_context *ce) +{ + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_GUC_ID_STOLEN); +} + +static inline void set_context_guc_id_stolen(struct intel_context *ce) +{ + atomic_or(SCHED_STATE_NO_LOCK_GUC_ID_STOLEN, + &ce->guc_sched_state_no_lock); +} + +static inline void clr_context_guc_id_stolen(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_GUC_ID_STOLEN, + &ce->guc_sched_state_no_lock); +} + +static inline bool context_needs_register(struct intel_context *ce) +{ + return (atomic_read(&ce->guc_sched_state_no_lock) & + SCHED_STATE_NO_LOCK_NEEDS_REGISTER); +} + +static inline void set_context_needs_register(struct intel_context *ce) +{ + atomic_or(SCHED_STATE_NO_LOCK_NEEDS_REGISTER, + &ce->guc_sched_state_no_lock); +} + +static inline void clr_context_needs_register(struct intel_context *ce) +{ + atomic_and((u32)~SCHED_STATE_NO_LOCK_NEEDS_REGISTER, + &ce->guc_sched_state_no_lock); +} + /* * Below is a set of functions which control the GuC scheduling state which * require a lock, aside from the special case where the functions are called @@ -418,9 +571,12 @@ int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout) true, timeout); }

-static int guc_lrc_desc_pin(struct intel_context *ce, bool loop); +static inline bool request_has_no_guc_id(struct i915_request *rq) +{ + return test_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); +}

-static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) +static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) { int err = 0; struct intel_context *ce = rq->context; @@ -439,18 +595,15 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) goto out; }

+ /* Ensure context is in correct state before a submission */ + GEM_BUG_ON(ce->guc_num_rq_submit_no_id); + GEM_BUG_ON(request_has_no_guc_id(rq)); GEM_BUG_ON(!atomic_read(&ce->guc_id_ref)); + GEM_BUG_ON(context_needs_register(ce)); GEM_BUG_ON(context_guc_id_invalid(ce)); - - /* - * Corner case where the GuC firmware was blown away and reloaded while - * this context was pinned. - */ - if (unlikely(!lrc_desc_registered(guc, ce->guc_id))) { - err = guc_lrc_desc_pin(ce, false); - if (unlikely(err)) - goto out; - } + GEM_BUG_ON(context_pending_disable(ce)); + GEM_BUG_ON(context_wait_for_deregister_to_register(ce)); + GEM_BUG_ON(!lrc_desc_registered(guc, ce->guc_id));

/* * The request / context will be run on the hardware when scheduling @@ -462,6 +615,8 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) enabled = context_enabled(ce);

if (!enabled) { + GEM_BUG_ON(context_pending_enable(ce)); + action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET; action[len++] = ce->guc_id; action[len++] = GUC_CONTEXT_ENABLE; @@ -489,6 +644,67 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) return err; }

+static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) +{ + int ret; + + lockdep_assert_held(&guc->sched_engine->lock); + + ret = __guc_add_request(guc, rq); + if (ret == -EBUSY) { + guc->stalled_rq = rq; + guc->submission_stall_reason = STALL_ADD_REQUEST; + } else { + guc->stalled_rq = NULL; + guc->submission_stall_reason = STALL_NONE; + } + + return ret; +} + +static int guc_lrc_desc_pin(struct intel_context *ce, bool loop); + +static int tasklet_register_context(struct intel_guc *guc, + struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + int ret = 0; + + /* Check state */ + lockdep_assert_held(&guc->sched_engine->lock); + GEM_BUG_ON(ce->guc_num_rq_submit_no_id); + GEM_BUG_ON(request_has_no_guc_id(rq)); + GEM_BUG_ON(context_guc_id_invalid(ce)); + GEM_BUG_ON(!atomic_read(&ce->guc_id_ref)); + + /* + * The guc_id is getting pinned during the tasklet and we need to + * register this context or a corner case where the GuC firmware was + * blown away and reloaded while this context was pinned + */ + if (unlikely((!lrc_desc_registered(guc, ce->guc_id) || + context_needs_register(ce)) && + !intel_context_is_banned(ce))) { + GEM_BUG_ON(context_pending_disable(ce)); + GEM_BUG_ON(context_wait_for_deregister_to_register(ce)); + + ret = guc_lrc_desc_pin(ce, false); + + if (likely(ret != -EBUSY)) + clr_context_needs_register(ce); + + if (unlikely(ret == -EBUSY)) { + guc->stalled_rq = rq; + guc->submission_stall_reason = STALL_REGISTER_CONTEXT; + } else if (unlikely(ret == -EINPROGRESS)) { + guc->stalled_rq = rq; + guc->submission_stall_reason = STALL_DEREGISTER_CONTEXT; + } + } + + return ret; +} + static inline void guc_set_lrc_tail(struct i915_request *rq) { rq->context->lrc_reg_state[CTX_RING_TAIL] = @@ -500,77 +716,142 @@ static inline int rq_prio(const struct i915_request *rq) return rq->sched.attr.priority; }

+static void kick_retire_wq(struct intel_guc *guc) +{ + queue_work(system_unbound_wq, &guc->retire_worker); +} + +static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq); + static int guc_dequeue_one_context(struct intel_guc *guc) { struct i915_sched_engine * const sched_engine = guc->sched_engine; - struct i915_request *last = NULL; - bool submit = false; + struct i915_request *last = guc->stalled_rq; + bool submit = !!last; struct rb_node *rb; int ret;

lockdep_assert_held(&sched_engine->lock); + GEM_BUG_ON(guc->stalled_context); + GEM_BUG_ON(!submit && guc->submission_stall_reason);

- if (guc->stalled_request) { - submit = true; - last = guc->stalled_request; - goto resubmit; - } + if (submit) { + /* Flow control conditions */ + switch (guc->submission_stall_reason) { + case STALL_GUC_ID_TASKLET: + goto done; + case STALL_REGISTER_CONTEXT: + goto register_context; + case STALL_MOVE_LRC_TAIL: + goto move_lrc_tail; + case STALL_ADD_REQUEST: + goto add_request; + default: + GEM_BUG_ON("Invalid stall state"); + } + } else { + GEM_BUG_ON(!guc->total_num_rq_with_no_guc_id && + guc_ids_exhausted(guc));

- while ((rb = rb_first_cached(&sched_engine->queue))) { - struct i915_priolist *p = to_priolist(rb); - struct i915_request *rq, *rn; + while ((rb = rb_first_cached(&sched_engine->queue))) { + struct i915_priolist *p = to_priolist(rb); + struct i915_request *rq, *rn;

- priolist_for_each_request_consume(rq, rn, p) { - if (last && rq->context != last->context) - goto done; + priolist_for_each_request_consume(rq, rn, p) { + if (last && rq->context != last->context) + goto done;

- list_del_init(&rq->sched.link); + list_del_init(&rq->sched.link);

- __i915_request_submit(rq); + __i915_request_submit(rq);

- trace_i915_request_in(rq, 0); - last = rq; - submit = true; - } + trace_i915_request_in(rq, 0); + last = rq; + submit = true; + }

- rb_erase_cached(&p->node, &sched_engine->queue); - i915_priolist_free(p); + rb_erase_cached(&p->node, &sched_engine->queue); + i915_priolist_free(p); + } } + done: if (submit) { + struct intel_context *ce = last->context; + + if (ce->guc_num_rq_submit_no_id) { + ret = tasklet_pin_guc_id(guc, last); + if (ret) + goto blk_tasklet_kick; + } + +register_context: + ret = tasklet_register_context(guc, last); + if (unlikely(ret == -EINPROGRESS)) { + goto blk_tasklet; + } else if (unlikely(ret == -EPIPE)) { + goto deadlk; + } else if (ret == -EBUSY) { + goto schedule_tasklet; + } else if (unlikely(ret != 0)) { + GEM_WARN_ON(ret); /* Unexpected */ + goto deadlk; + } + +move_lrc_tail: guc_set_lrc_tail(last); -resubmit: + +add_request: ret = guc_add_request(guc, last); - if (unlikely(ret == -EPIPE)) + if (unlikely(ret == -EPIPE)) { + goto deadlk; + } else if (ret == -EBUSY) { + goto schedule_tasklet; + } else if (unlikely(ret != 0)) { + GEM_WARN_ON(ret); /* Unexpected */ goto deadlk; - else if (ret == -EBUSY) { - tasklet_schedule(&sched_engine->tasklet); - guc->stalled_request = last; - return false; } }

- guc->stalled_request = NULL; + /* + * No requests without a guc_id, enable guc_id allocation at request + * creation time (guc_request_alloc). + */ + if (!guc->total_num_rq_with_no_guc_id) + clr_guc_ids_exhausted(guc); + return submit;

+schedule_tasklet: + tasklet_schedule(&sched_engine->tasklet); + return false; + deadlk: sched_engine->tasklet.callback = NULL; tasklet_disable_nosync(&sched_engine->tasklet); return false; + +blk_tasklet_kick: + kick_retire_wq(guc); +blk_tasklet: + set_tasklet_blocked(guc); + return false; }

static void guc_submission_tasklet(struct tasklet_struct *t) { struct i915_sched_engine *sched_engine = from_tasklet(sched_engine, t, tasklet); + struct intel_guc *guc = sched_engine->private_data; unsigned long flags; bool loop;

spin_lock_irqsave(&sched_engine->lock, flags);

- do { - loop = guc_dequeue_one_context(sched_engine->private_data); - } while (loop); + if (likely(!tasklet_blocked(guc))) + do { + loop = guc_dequeue_one_context(guc); + } while (loop);

i915_sched_engine_reset_on_empty(sched_engine);

@@ -653,6 +934,14 @@ submission_disabled(struct intel_guc *guc) !__tasklet_is_enabled(&sched_engine->tasklet)); }

+static void kick_tasklet(struct intel_guc *guc) +{ + struct i915_sched_engine * const sched_engine = guc->sched_engine; + + if (likely(!tasklet_blocked(guc))) + tasklet_hi_schedule(&sched_engine->tasklet); +} + static void disable_submission(struct intel_guc *guc) { struct i915_sched_engine * const sched_engine = guc->sched_engine; @@ -676,8 +965,16 @@ static void enable_submission(struct intel_guc *guc) __tasklet_enable(&sched_engine->tasklet)) { GEM_BUG_ON(!guc->ct.enabled);

+ /* Reset tasklet state */ + guc->stalled_rq = NULL; + if (guc->stalled_context) + intel_context_put(guc->stalled_context); + guc->stalled_context = NULL; + guc->submission_stall_reason = STALL_NONE; + guc->flags = 0; + /* And kick in case we missed a new request submission. */ - tasklet_hi_schedule(&sched_engine->tasklet); + kick_tasklet(guc); } spin_unlock_irqrestore(&guc->sched_engine->lock, flags); } @@ -856,6 +1153,7 @@ static void __guc_reset_context(struct intel_context *ce, bool stalled) out_replay: guc_reset_state(ce, head, stalled); __unwind_incomplete_requests(ce); + ce->guc_num_rq_submit_no_id = 0; intel_context_put(ce); }

@@ -888,6 +1186,7 @@ static void guc_cancel_context_requests(struct intel_context *ce) spin_lock(&ce->guc_active.lock); list_for_each_entry(rq, &ce->guc_active.requests, sched.link) i915_request_put(i915_request_mark_eio(rq)); + ce->guc_num_rq_submit_no_id = 0; spin_unlock(&ce->guc_active.lock); spin_unlock_irqrestore(&sched_engine->lock, flags); } @@ -924,11 +1223,15 @@ guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine) struct i915_priolist *p = to_priolist(rb);

priolist_for_each_request_consume(rq, rn, p) { + struct intel_context *ce = rq->context; + list_del_init(&rq->sched.link);

__i915_request_submit(rq);

i915_request_put(i915_request_mark_eio(rq)); + + ce->guc_num_rq_submit_no_id = 0; }

rb_erase_cached(&p->node, &sched_engine->queue); @@ -980,6 +1283,51 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) intel_gt_unpark_heartbeats(guc_to_gt(guc)); }

+static void retire_worker_sched_disable(struct intel_guc *guc, + struct intel_context *ce); + +static void retire_worker_func(struct work_struct *w) +{ + struct intel_guc *guc = + container_of(w, struct intel_guc, retire_worker); + + /* + * It is possible that another thread issues the schedule disable + that + * G2H completes moving the state machine further along to a point + * where nothing needs to be done here. Let's be paranoid and kick the + * tasklet in that case. + */ + if (guc->submission_stall_reason != STALL_SCHED_DISABLE && + guc->submission_stall_reason != STALL_GUC_ID_WORKQUEUE) { + kick_tasklet(guc); + return; + } + + if (guc->submission_stall_reason == STALL_SCHED_DISABLE) { + GEM_BUG_ON(!guc->stalled_context); + GEM_BUG_ON(context_guc_id_invalid(guc->stalled_context)); + + retire_worker_sched_disable(guc, guc->stalled_context); + } + + /* + * guc_id pressure, always try to release it regardless of state, + * albeit after possibly issuing a schedule disable as that is async + * operation. + */ + intel_gt_retire_requests(guc_to_gt(guc)); + + if (guc->submission_stall_reason == STALL_GUC_ID_WORKQUEUE) { + GEM_BUG_ON(guc->stalled_context); + + /* Hopefully guc_ids are now available, kick tasklet */ + guc->submission_stall_reason = STALL_GUC_ID_TASKLET; + clr_tasklet_blocked(guc); + + kick_tasklet(guc); + } +} + /* * Set up the memory resources to be shared with the GuC (via the GGTT) * at firmware loading time. @@ -1003,9 +1351,12 @@ int intel_guc_submission_init(struct intel_guc *guc) xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);

spin_lock_init(&guc->contexts_lock); - INIT_LIST_HEAD(&guc->guc_id_list); + INIT_LIST_HEAD(&guc->guc_id_list_no_ref); + INIT_LIST_HEAD(&guc->guc_id_list_unpinned); ida_init(&guc->guc_ids);

+ INIT_WORK(&guc->retire_worker, retire_worker_func); + return 0; }

@@ -1022,10 +1373,28 @@ static inline void queue_request(struct i915_sched_engine *sched_engine, struct i915_request *rq, int prio) { + bool empty = i915_sched_engine_is_empty(sched_engine); + GEM_BUG_ON(!list_empty(&rq->sched.link)); list_add_tail(&rq->sched.link, i915_sched_lookup_priolist(sched_engine, prio)); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); + + if (empty) + kick_tasklet(&rq->engine->gt->uc.guc); +} + +static bool need_tasklet(struct intel_guc *guc, struct intel_context *ce) +{ + struct i915_sched_engine * const sched_engine = + ce->engine->sched_engine; + + lockdep_assert_held(&sched_engine->lock); + + return guc_ids_exhausted(guc) || submission_disabled(guc) || + guc->stalled_rq || guc->stalled_context || + !lrc_desc_registered(guc, ce->guc_id) || + !i915_sched_engine_is_empty(sched_engine); }

static int guc_bypass_tasklet_submit(struct intel_guc *guc, @@ -1039,8 +1408,6 @@ static int guc_bypass_tasklet_submit(struct intel_guc *guc,

guc_set_lrc_tail(rq); ret = guc_add_request(guc, rq); - if (ret == -EBUSY) - guc->stalled_request = rq;

if (unlikely(ret == -EPIPE)) disable_submission(guc); @@ -1057,11 +1424,10 @@ static void guc_submit_request(struct i915_request *rq) /* Will be called from irq-context when using foreign fences. */ spin_lock_irqsave(&sched_engine->lock, flags);

- if (submission_disabled(guc) || guc->stalled_request || - !i915_sched_engine_is_empty(sched_engine)) + if (need_tasklet(guc, rq->context)) queue_request(sched_engine, rq, rq_prio(rq)); else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY) - tasklet_hi_schedule(&sched_engine->tasklet); + kick_tasklet(guc);

spin_unlock_irqrestore(&sched_engine->lock, flags); } @@ -1093,32 +1459,71 @@ static void release_guc_id(struct intel_guc *guc, struct intel_context *ce) spin_unlock_irqrestore(&guc->contexts_lock, flags); }

-static int steal_guc_id(struct intel_guc *guc) +/* + * We have two lists for guc_ids available to steal. One list is for contexts + * that to have a zero guc_id_ref but are still pinned (scheduling enabled, only + * available inside tasklet) and the other is for contexts that are not pinned + * but still registered (available both outside and inside tasklet). Stealing + * from the latter only requires a deregister H2G, while the former requires a + * schedule disable H2G + a deregister H2G. + */ +static struct list_head *get_guc_id_list(struct intel_guc *guc, + bool unpinned) +{ + if (unpinned) + return &guc->guc_id_list_unpinned; + else + return &guc->guc_id_list_no_ref; +} + +static int steal_guc_id(struct intel_guc *guc, bool unpinned) { struct intel_context *ce; int guc_id; + struct list_head *guc_id_list = get_guc_id_list(guc, unpinned);

lockdep_assert_held(&guc->contexts_lock);

- if (!list_empty(&guc->guc_id_list)) { - ce = list_first_entry(&guc->guc_id_list, + if (!list_empty(guc_id_list)) { + ce = list_first_entry(guc_id_list, struct intel_context, guc_id_link);

+ /* Ensure context getting stolen in expected state */ GEM_BUG_ON(atomic_read(&ce->guc_id_ref)); GEM_BUG_ON(context_guc_id_invalid(ce)); + GEM_BUG_ON(context_guc_id_stolen(ce));

list_del_init(&ce->guc_id_link); guc_id = ce->guc_id; clr_context_registered(ce); - set_context_guc_id_invalid(ce); + + /* + * If stealing from the pinned list, defer invalidating + * the guc_id until the retire workqueue processes this + * context. + */ + if (!unpinned) { + GEM_BUG_ON(guc->stalled_context); + guc->stalled_context = intel_context_get(ce); + set_context_guc_id_stolen(ce); + } else { + set_context_guc_id_invalid(ce); + } + return guc_id; } else { return -EAGAIN; } }

-static int assign_guc_id(struct intel_guc *guc, u16 *out) +enum { /* Return values for pin_guc_id / assign_guc_id */ + SAME_GUC_ID = 0, + NEW_GUC_ID_DISABLED = 1, + NEW_GUC_ID_ENABLED = 2, +}; + +static int assign_guc_id(struct intel_guc *guc, u16 *out, bool tasklet) { int ret;

@@ -1126,17 +1531,33 @@ static int assign_guc_id(struct intel_guc *guc, u16 *out)

ret = new_guc_id(guc); if (unlikely(ret < 0)) { - ret = steal_guc_id(guc); - if (ret < 0) - return ret; + ret = steal_guc_id(guc, true); + if (ret >= 0) { + *out = ret; + ret = NEW_GUC_ID_DISABLED; + } else if (ret < 0 && tasklet) { + /* + * We only steal a guc_id from a context with scheduling + * enabled if guc_ids are exhausted and we are submitting + * from the tasklet. + */ + ret = steal_guc_id(guc, false); + if (ret >= 0) { + *out = ret; + ret = NEW_GUC_ID_ENABLED; + } + } + } else { + *out = ret; + ret = SAME_GUC_ID; }

- *out = ret; - return 0; + return ret; }

#define PIN_GUC_ID_TRIES 4 -static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) +static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce, + bool tasklet) { int ret = 0; unsigned long flags, tries = PIN_GUC_ID_TRIES; @@ -1146,11 +1567,15 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) try_again: spin_lock_irqsave(&guc->contexts_lock, flags);

+ if (!tasklet && guc_ids_exhausted(guc)) { + ret = -EAGAIN; + goto out_unlock; + } + if (context_guc_id_invalid(ce)) { - ret = assign_guc_id(guc, &ce->guc_id); - if (ret) + ret = assign_guc_id(guc, &ce->guc_id, tasklet); + if (unlikely(ret < 0)) goto out_unlock; - ret = 1; /* Indidcates newly assigned guc_id */ } if (!list_empty(&ce->guc_id_link)) list_del_init(&ce->guc_id_link); @@ -1166,8 +1591,11 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) * attempting to retire more requests. Double the sleep period each * subsequent pass before finally giving up. The sleep period has max of * 100ms and minimum of 1ms. + * + * We only try this if outside the tasklet, inside the tasklet we have a + * (slower, more complex, blocking) different flow control algorithm. */ - if (ret == -EAGAIN && --tries) { + if (ret == -EAGAIN && --tries && !tasklet) { if (PIN_GUC_ID_TRIES - tries > 1) { unsigned int timeslice_shifted = ce->engine->props.timeslice_duration_ms << @@ -1184,7 +1612,9 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce) return ret; }

-static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce) +static void unpin_guc_id(struct intel_guc *guc, + struct intel_context *ce, + bool unpinned) { unsigned long flags;

@@ -1194,9 +1624,17 @@ static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce) return;

spin_lock_irqsave(&guc->contexts_lock, flags); - if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id_link) && - !atomic_read(&ce->guc_id_ref)) - list_add_tail(&ce->guc_id_link, &guc->guc_id_list); + + if (!list_empty(&ce->guc_id_link)) + list_del_init(&ce->guc_id_link); + + if (!context_guc_id_invalid(ce) && !context_guc_id_stolen(ce) && + !atomic_read(&ce->guc_id_ref)) { + struct list_head *head = get_guc_id_list(guc, unpinned); + + list_add_tail(&ce->guc_id_link, head); + } + spin_unlock_irqrestore(&guc->contexts_lock, flags); }

@@ -1300,6 +1738,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) int ret = 0;

GEM_BUG_ON(!engine->mask); + GEM_BUG_ON(context_guc_id_invalid(ce));

/* * Ensure LRC + CT vmas are is same region as write barrier is done @@ -1342,6 +1781,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) trace_intel_context_steal_guc_id(ce); if (!loop) { set_context_wait_for_deregister_to_register(ce); + set_context_block_tasklet(ce); intel_context_get(ce); } else { bool disabled; @@ -1369,7 +1809,14 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) ret = deregister_context(ce, ce->guc_id, loop); if (unlikely(ret == -EBUSY)) { clr_context_wait_for_deregister_to_register(ce); + clr_context_block_tasklet(ce); intel_context_put(ce); + } else if (!loop && !ret) { + /* + * A context de-registration has been issued from within + * the tasklet. Need to block until it complete. + */ + return -EINPROGRESS; } else if (unlikely(ret == -ENODEV)) { ret = 0; /* Will get registered later */ } @@ -1425,7 +1872,9 @@ static void guc_context_unpin(struct intel_context *ce) { struct intel_guc *guc = ce_to_guc(ce);

- unpin_guc_id(guc, ce); + GEM_BUG_ON(context_enabled(ce)); + + unpin_guc_id(guc, ce, true); lrc_unpin(ce); }

@@ -1764,6 +2213,8 @@ static void guc_context_destroy(struct kref *kref) unsigned long flags; bool disabled;

+ GEM_BUG_ON(context_guc_id_stolen(ce)); + /* * If the guc_id is invalid this context has been stolen and we can free * it immediately. Also can be freed immediately if the context is not @@ -1925,6 +2376,9 @@ static void add_to_context(struct i915_request *rq) spin_lock(&ce->guc_active.lock); list_move_tail(&rq->sched.link, &ce->guc_active.requests);

+ if (unlikely(request_has_no_guc_id(rq))) + ++ce->guc_num_rq_submit_no_id; + if (rq->guc_prio == GUC_PRIO_INIT) { rq->guc_prio = new_guc_prio; add_context_inflight_prio(ce, rq->guc_prio); @@ -1966,7 +2420,12 @@ static void remove_from_context(struct i915_request *rq)

spin_unlock_irq(&ce->guc_active.lock);

- atomic_dec(&ce->guc_id_ref); + if (likely(!request_has_no_guc_id(rq))) + atomic_dec(&ce->guc_id_ref); + else + --ce_to_guc(rq->context)->total_num_rq_with_no_guc_id; + unpin_guc_id(ce_to_guc(ce), ce, false); + i915_request_notify_execute_cb_imm(rq); }

@@ -2018,13 +2477,144 @@ static void guc_signal_context_fence(struct intel_context *ce) spin_unlock_irqrestore(&ce->guc_state.lock, flags); }

-static bool context_needs_register(struct intel_context *ce, bool new_guc_id) +static void invalidate_guc_id_sched_disable(struct intel_context *ce) +{ + set_context_guc_id_invalid(ce); + wmb(); /* Make sure guc_id invalidation visible first */ + clr_context_guc_id_stolen(ce); +} + +static void retire_worker_sched_disable(struct intel_guc *guc, + struct intel_context *ce) +{ + unsigned long flags; + bool disabled; + + guc->stalled_context = NULL; + spin_lock_irqsave(&ce->guc_state.lock, flags); + disabled = submission_disabled(guc); + if (!disabled && !context_pending_disable(ce) && context_enabled(ce)) { + /* + * Still enabled, issue schedule disable + configure state so + * when G2H returns tasklet is kicked. + */ + + struct intel_runtime_pm *runtime_pm = + &ce->engine->gt->i915->runtime_pm; + intel_wakeref_t wakeref; + u16 guc_id; + + /* + * We add +2 here as the schedule disable complete CTB handler + * calls intel_context_sched_disable_unpin (-2 to pin_count). + */ + GEM_BUG_ON(!atomic_read(&ce->pin_count)); + atomic_add(2, &ce->pin_count); + + set_context_block_tasklet(ce); + guc_id = prep_context_pending_disable(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + with_intel_runtime_pm(runtime_pm, wakeref) + __guc_context_sched_disable(guc, ce, guc_id); + + invalidate_guc_id_sched_disable(ce); + } else if (!disabled && context_pending_disable(ce)) { + /* + * Schedule disable in flight, set bit to kick tasklet in G2H + * handler and call it a day. + */ + + set_context_block_tasklet(ce); + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + invalidate_guc_id_sched_disable(ce); + } else { + /* Schedule disable is done, kick tasklet */ + + spin_unlock_irqrestore(&ce->guc_state.lock, flags); + + invalidate_guc_id_sched_disable(ce); + + guc->submission_stall_reason = STALL_REGISTER_CONTEXT; + clr_tasklet_blocked(guc); + + kick_tasklet(ce_to_guc(ce)); + } + + intel_context_put(ce); +} + +static bool context_needs_lrc_desc_pin(struct intel_context *ce, bool new_guc_id) { return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) || !lrc_desc_registered(ce_to_guc(ce), ce->guc_id)) && !submission_disabled(ce_to_guc(ce)); }

+static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + int ret = 0; + + lockdep_assert_held(&guc->sched_engine->lock); + GEM_BUG_ON(!ce->guc_num_rq_submit_no_id); + + if (atomic_add_unless(&ce->guc_id_ref, ce->guc_num_rq_submit_no_id, 0)) + goto out; + + ret = pin_guc_id(guc, ce, true); + if (unlikely(ret < 0)) { + /* + * No guc_ids available, disable the tasklet and kick the retire + * workqueue hopefully freeing up some guc_ids. + */ + guc->stalled_rq = rq; + guc->submission_stall_reason = STALL_GUC_ID_WORKQUEUE; + return ret; + } + + if (ce->guc_num_rq_submit_no_id - 1 > 0) + atomic_add(ce->guc_num_rq_submit_no_id - 1, + &ce->guc_id_ref); + + if (context_needs_lrc_desc_pin(ce, !!ret)) + set_context_needs_register(ce); + + if (ret == NEW_GUC_ID_ENABLED) { + guc->stalled_rq = rq; + guc->submission_stall_reason = STALL_SCHED_DISABLE; + } + + clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags); +out: + guc->total_num_rq_with_no_guc_id -= ce->guc_num_rq_submit_no_id; + GEM_BUG_ON(guc->total_num_rq_with_no_guc_id < 0); + + list_for_each_entry_reverse(rq, &ce->guc_active.requests, sched.link) + if (request_has_no_guc_id(rq)) { + --ce->guc_num_rq_submit_no_id; + clear_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, + &rq->fence.flags); + } else if (!ce->guc_num_rq_submit_no_id) { + break; + } + + GEM_BUG_ON(ce->guc_num_rq_submit_no_id); + + /* + * When NEW_GUC_ID_ENABLED is returned it means we are stealing a guc_id + * from a context that has scheduling enabled. We have to disable + * scheduling before deregistering the context and it isn't safe to do + * in the tasklet because of lock inversion (ce->guc_state.lock must be + * acquired before guc->sched_engine->lock). To work around this + * we do the schedule disable in retire workqueue and block the tasklet + * until the schedule done G2H returns. Returning non-zero here kicks + * the workqueue. + */ + return (ret == NEW_GUC_ID_ENABLED) ? ret : 0; +} + static int guc_request_alloc(struct i915_request *rq) { struct intel_context *ce = rq->context; @@ -2056,6 +2646,15 @@ static int guc_request_alloc(struct i915_request *rq)

rq->reserved_space -= GUC_REQUEST_SIZE;

+ /* + * guc_ids are exhausted, don't allocate one here, defer to submission + * in the tasklet. + */ + if (test_and_update_guc_ids_exhausted(guc)) { + set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); + goto out; + } + /* * Call pin_guc_id here rather than in the pinning step as with * dma_resv, contexts can be repeatedly pinned / unpinned trashing the @@ -2063,9 +2662,7 @@ static int guc_request_alloc(struct i915_request *rq) * when guc_ids are being stolen due to over subscription. By the time * this function is reached, it is guaranteed that the guc_id will be * persistent until the generated request is retired. Thus, sealing these - * race conditions. It is still safe to fail here if guc_ids are - * exhausted and return -EAGAIN to the user indicating that they can try - * again in the future. + * race conditions. * * There is no need for a lock here as the timeline mutex ensures at * most one context can be executing this code path at once. The @@ -2076,10 +2673,26 @@ static int guc_request_alloc(struct i915_request *rq) if (atomic_add_unless(&ce->guc_id_ref, 1, 0)) goto out;

- ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */ - if (unlikely(ret < 0)) + ret = pin_guc_id(guc, ce, false); /* > 0 indicates new guc_id */ + if (unlikely(ret == -EAGAIN)) { + /* + * No guc_ids available, so we force this submission and all + * future submissions to be serialized in the tasklet, sharing + * the guc_ids on a per submission basis to ensure (more) fair + * scheduling of submissions. Once the tasklet is flushed of + * submissions we return to allocating guc_ids in this function. + */ + set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); + set_and_update_guc_ids_exhausted(guc); + + return 0; + } else if (unlikely(ret < 0)) { return ret; - if (context_needs_register(ce, !!ret)) { + } + + GEM_BUG_ON(ret == NEW_GUC_ID_ENABLED); + + if (context_needs_lrc_desc_pin(ce, !!ret)) { ret = guc_lrc_desc_pin(ce, true); if (unlikely(ret)) { /* unwind */ if (ret == -EPIPE) { @@ -2087,7 +2700,7 @@ static int guc_request_alloc(struct i915_request *rq) goto out; /* GPU will be reset */ } atomic_dec(&ce->guc_id_ref); - unpin_guc_id(guc, ce); + unpin_guc_id(guc, ce, true); return ret; } } @@ -2358,7 +2971,7 @@ static inline void guc_kernel_context_pin(struct intel_guc *guc, struct intel_context *ce) { if (context_guc_id_invalid(ce)) - pin_guc_id(guc, ce); + pin_guc_id(guc, ce, false); guc_lrc_desc_pin(ce, true); }

@@ -2625,6 +3238,16 @@ int intel_guc_deregister_done_process_msg(struct intel_guc *guc, with_intel_runtime_pm(runtime_pm, wakeref) register_context(ce, true); guc_signal_context_fence(ce); + if (context_block_tasklet(ce)) { + GEM_BUG_ON(guc->submission_stall_reason != + STALL_DEREGISTER_CONTEXT); + + clr_context_block_tasklet(ce); + guc->submission_stall_reason = STALL_MOVE_LRC_TAIL; + clr_tasklet_blocked(guc); + + kick_tasklet(ce_to_guc(ce)); + } intel_context_put(ce); } else if (context_destroyed(ce)) { /* Context has been destroyed */ @@ -2688,6 +3311,14 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, guc_blocked_fence_complete(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags);

+ if (context_block_tasklet(ce)) { + clr_context_block_tasklet(ce); + guc->submission_stall_reason = STALL_REGISTER_CONTEXT; + clr_tasklet_blocked(guc); + + kick_tasklet(ce_to_guc(ce)); + } + if (banned) { guc_cancel_context_requests(ce); intel_engine_signal_breadcrumbs(ce->engine); @@ -2716,10 +3347,8 @@ static void capture_error_state(struct intel_guc *guc,

static void guc_context_replay(struct intel_context *ce) { - struct i915_sched_engine *sched_engine = ce->engine->sched_engine; - __guc_reset_context(ce, true); - tasklet_hi_schedule(&sched_engine->tasklet); + kick_tasklet(ce_to_guc(ce)); }

static void guc_handle_context_reset(struct intel_guc *guc, @@ -2878,8 +3507,16 @@ void intel_guc_submission_print_info(struct intel_guc *guc, atomic_read(&guc->outstanding_submission_g2h)); drm_printf(p, "GuC Number GuC IDs: %u\n", guc->num_guc_ids); drm_printf(p, "GuC Max GuC IDs: %u\n", guc->max_guc_ids); - drm_printf(p, "GuC tasklet count: %u\n\n", + drm_printf(p, "GuC tasklet count: %u\n", atomic_read(&sched_engine->tasklet.count)); + drm_printf(p, "GuC submit flags: 0x%04lx\n", guc->flags); + drm_printf(p, "GuC total number request without guc_id: %d\n", + guc->total_num_rq_with_no_guc_id); + drm_printf(p, "GuC stall reason: %d\n", guc->submission_stall_reason); + drm_printf(p, "GuC stalled request: %s\n", + yesno(guc->stalled_rq)); + drm_printf(p, "GuC stalled context: %s\n\n", + yesno(guc->stalled_context));

spin_lock_irqsave(&sched_engine->lock, flags); drm_printf(p, "Requests in GuC submit tasklet:\n"); diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 1bc1349ba3c2..807f76750cf4 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -139,6 +139,12 @@ enum { * the GPU. Here we track such boost requests on a per-request basis. */ I915_FENCE_FLAG_BOOST, + + /* + * I915_FENCE_FLAG_GUC_ID_NOT_PINNED - Set to signal the GuC submission + * tasklet that the guc_id isn't pinned. + */ + I915_FENCE_FLAG_GUC_ID_NOT_PINNED, };

/**

Add a heuristic which checks if over half of the available guc_ids are currently consumed by requests not ready to be submitted. If this heuristic is true at request creation time (normal guc_id allocation location) force all submissions + guc_ids allocations to tasklet.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_context_types.h | 3 ++ drivers/gpu/drm/i915/gt/intel_reset.c | 9 ++++ drivers/gpu/drm/i915/gt/uc/intel_guc.h | 1 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 53 +++++++++++++++++-- .../gpu/drm/i915/gt/uc/intel_guc_submission.h | 2 + 5 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index 8ed964ef967b..c01530d7dc67 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -188,6 +188,9 @@ struct intel_context { /* Number of rq submitted without a guc_id */ u16 guc_num_rq_submit_no_id;

+ /* GuC number of requests not ready */ + atomic_t guc_num_rq_not_ready; + /* * GuC ID link - in list when unpinned but guc_id still valid in GuC */ diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 91200c43951f..ea763138197f 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -22,6 +22,7 @@ #include "intel_reset.h"

#include "uc/intel_guc.h" +#include "uc/intel_guc_submission.h"

#define RESET_MAX_RETRIES 3

@@ -850,6 +851,14 @@ static void nop_submit_request(struct i915_request *request) { RQ_TRACE(request, "-EIO\n");

+ /* + * XXX: Kinda ugly to check for GuC submission here but this function is + * going away once we switch to the DRM scheduler so we can live with + * this for now. + */ + if (intel_engine_uses_guc(request->engine)) + intel_guc_decr_num_rq_not_ready(request->context); + request = i915_request_mark_eio(request); if (request) { i915_request_submit(request); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index e76579396efd..917352c9f323 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -76,6 +76,7 @@ struct intel_guc { struct ida guc_ids; u32 num_guc_ids; u32 max_guc_ids; + atomic_t num_guc_ids_not_ready; struct list_head guc_id_list_no_ref; struct list_head guc_id_list_unpinned;

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index f42a707f60ca..ba750fc87af1 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1384,6 +1384,41 @@ static inline void queue_request(struct i915_sched_engine *sched_engine, kick_tasklet(&rq->engine->gt->uc.guc); }

+/* Macro to tweak heuristic, using a simple over 50% not ready for now */ +#define TOO_MANY_GUC_IDS_NOT_READY(avail, consumed) \ + ((consumed) > (avail) / 2) +static bool too_many_guc_ids_not_ready(struct intel_guc *guc, + struct intel_context *ce) +{ + u32 available_guc_ids, guc_ids_consumed; + + available_guc_ids = guc->num_guc_ids; + guc_ids_consumed = atomic_read(&guc->num_guc_ids_not_ready); + + if (TOO_MANY_GUC_IDS_NOT_READY(available_guc_ids, guc_ids_consumed)) { + set_and_update_guc_ids_exhausted(guc); + return true; + } + + return false; +} + +static void incr_num_rq_not_ready(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + + if (!atomic_fetch_add(1, &ce->guc_num_rq_not_ready)) + atomic_inc(&guc->num_guc_ids_not_ready); +} + +void intel_guc_decr_num_rq_not_ready(struct intel_context *ce) +{ + struct intel_guc *guc = ce_to_guc(ce); + + if (atomic_fetch_add(-1, &ce->guc_num_rq_not_ready) == 1) + atomic_dec(&guc->num_guc_ids_not_ready); +} + static bool need_tasklet(struct intel_guc *guc, struct intel_context *ce) { struct i915_sched_engine * const sched_engine = @@ -1430,6 +1465,8 @@ static void guc_submit_request(struct i915_request *rq) kick_tasklet(guc);

spin_unlock_irqrestore(&sched_engine->lock, flags); + + intel_guc_decr_num_rq_not_ready(rq->context); }

static int new_guc_id(struct intel_guc *guc) @@ -2647,10 +2684,13 @@ static int guc_request_alloc(struct i915_request *rq) rq->reserved_space -= GUC_REQUEST_SIZE;

/* - * guc_ids are exhausted, don't allocate one here, defer to submission - * in the tasklet. + * guc_ids are exhausted or a heuristic is met indicating too many + * guc_ids are waiting on requests with submission dependencies (not + * ready to submit). Don't allocate one here, defer to submission in the + * tasklet. */ - if (test_and_update_guc_ids_exhausted(guc)) { + if (test_and_update_guc_ids_exhausted(guc) || + too_many_guc_ids_not_ready(guc, ce)) { set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); goto out; } @@ -2684,6 +2724,7 @@ static int guc_request_alloc(struct i915_request *rq) */ set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); set_and_update_guc_ids_exhausted(guc); + incr_num_rq_not_ready(ce);

return 0; } else if (unlikely(ret < 0)) { @@ -2708,6 +2749,8 @@ static int guc_request_alloc(struct i915_request *rq) clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags);

out: + incr_num_rq_not_ready(ce); + /* * We block all requests on this context if a G2H is pending for a * schedule disable or context deregistration as the GuC will fail a @@ -3512,6 +3555,8 @@ void intel_guc_submission_print_info(struct intel_guc *guc, drm_printf(p, "GuC submit flags: 0x%04lx\n", guc->flags); drm_printf(p, "GuC total number request without guc_id: %d\n", guc->total_num_rq_with_no_guc_id); + drm_printf(p, "GuC Number GuC IDs not ready: %d\n", + atomic_read(&guc->num_guc_ids_not_ready)); drm_printf(p, "GuC stall reason: %d\n", guc->submission_stall_reason); drm_printf(p, "GuC stalled request: %s\n", yesno(guc->stalled_rq)); @@ -3567,6 +3612,8 @@ void intel_guc_submission_print_context_info(struct intel_guc *guc, atomic_read(&ce->pin_count)); drm_printf(p, "\t\tGuC ID Ref Count: %u\n", atomic_read(&ce->guc_id_ref)); + drm_printf(p, "\t\tNumber Requests Not Ready: %u\n", + atomic_read(&ce->guc_num_rq_not_ready)); drm_printf(p, "\t\tSchedule State: 0x%x, 0x%x\n\n", ce->guc_state.sched_state, atomic_read(&ce->guc_sched_state_no_lock)); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h index c7ef44fa0c36..17af5e123b09 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h @@ -51,4 +51,6 @@ static inline bool intel_guc_submission_is_used(struct intel_guc *guc) return intel_guc_is_used(guc) && intel_guc_submission_is_wanted(guc); }

+void intel_guc_decr_num_rq_not_ready(struct intel_context *ce); + #endif

Move fields related to controlling the GuC submission state machine to a unique object (guc_submit_engine) rather than the global GuC state (intel_guc). This encapsulation allows multiple instances of submission objects to operate in parallel and a single instance can block if needed while another can make forward progress. This is analogous to how the execlist mode works assigning a schedule object per physical engine but rather in GuC mode we assign a schedule object based on the blocking dependencies.

The guc_submit_engine object also encapsulates the i915_sched_engine object as well.

Lots of find-replace.

Currently only 1 guc_submit_engine instantiated, future patches will instantiate more.

Signed-off-by: Matthew Brost matthew.brost@intel.com Cc: John Harrison John.C.Harrison@Intel.com --- drivers/gpu/drm/i915/gt/uc/intel_guc.h | 33 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 557 +++++++++++------- .../i915/gt/uc/intel_guc_submission_types.h | 52 ++ drivers/gpu/drm/i915/i915_scheduler.c | 22 +- drivers/gpu/drm/i915/i915_scheduler.h | 3 + 5 files changed, 410 insertions(+), 257 deletions(-) create mode 100644 drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 917352c9f323..8ac016201658 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -21,6 +21,11 @@

struct __guc_ads_blob;

+enum { + GUC_SUBMIT_ENGINE_SINGLE_LRC, + GUC_SUBMIT_ENGINE_MAX +}; + /* * Top level structure of GuC. It handles firmware loading and manages client * pool. intel_guc owns a intel_guc_client to replace the legacy ExecList @@ -31,31 +36,6 @@ struct intel_guc { struct intel_guc_log log; struct intel_guc_ct ct;

- /* Global engine used to submit requests to GuC */ - struct i915_sched_engine *sched_engine; - - /* Global state related to submission tasklet */ - struct i915_request *stalled_rq; - struct intel_context *stalled_context; - struct work_struct retire_worker; - unsigned long flags; - int total_num_rq_with_no_guc_id; - - /* - * Submisson stall reason. See intel_guc_submission.c for detailed - * description. - */ - enum { - STALL_NONE, - STALL_GUC_ID_WORKQUEUE, - STALL_GUC_ID_TASKLET, - STALL_SCHED_DISABLE, - STALL_REGISTER_CONTEXT, - STALL_DEREGISTER_CONTEXT, - STALL_MOVE_LRC_TAIL, - STALL_ADD_REQUEST, - } submission_stall_reason; - /* intel_guc_recv interrupt related state */ spinlock_t irq_lock; unsigned int msg_enabled_mask; @@ -68,6 +48,8 @@ struct intel_guc { void (*disable)(struct intel_guc *guc); } interrupts;

+ struct guc_submit_engine *gse[GUC_SUBMIT_ENGINE_MAX]; + /* * contexts_lock protects the pool of free guc ids and a linked list of * guc ids available to be stolen @@ -76,7 +58,6 @@ struct intel_guc { struct ida guc_ids; u32 num_guc_ids; u32 max_guc_ids; - atomic_t num_guc_ids_not_ready; struct list_head guc_id_list_no_ref; struct list_head guc_id_list_unpinned;

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index ba750fc87af1..842094de848d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -21,6 +21,7 @@ #include "gt/intel_ring.h"

#include "intel_guc_submission.h" +#include "intel_guc_submission_types.h"

#include "i915_drv.h" #include "i915_trace.h" @@ -57,7 +58,7 @@ * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which * represents in-order queue. The kernel driver packs ring tail pointer and an * ELSP context descriptor dword into Work Item. - * See guc_add_request() + * See gse_add_request() * * GuC flow control state machine: * The tasklet, workqueue (retire_worker), and the G2H handlers together more or @@ -80,57 +81,57 @@ * context) */

-/* GuC Virtual Engine */ -struct guc_virtual_engine { - struct intel_engine_cs base; - struct intel_context context; -}; - static struct intel_context * guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count);

#define GUC_REQUEST_SIZE 64 /* bytes */

+static inline struct guc_submit_engine *ce_to_gse(struct intel_context *ce) +{ + return container_of(ce->engine->sched_engine, struct guc_submit_engine, + sched_engine); +} + /* * Global GuC flags helper functions */ enum { - GUC_STATE_TASKLET_BLOCKED, - GUC_STATE_GUC_IDS_EXHAUSTED, + GSE_STATE_TASKLET_BLOCKED, + GSE_STATE_GUC_IDS_EXHAUSTED, };

-static bool tasklet_blocked(struct intel_guc *guc) +static bool tasklet_blocked(struct guc_submit_engine *gse) { - return test_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); + return test_bit(GSE_STATE_TASKLET_BLOCKED, &gse->flags); }

-static void set_tasklet_blocked(struct intel_guc *guc) +static void set_tasklet_blocked(struct guc_submit_engine *gse) { - lockdep_assert_held(&guc->sched_engine->lock); - set_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); + lockdep_assert_held(&gse->sched_engine.lock); + set_bit(GSE_STATE_TASKLET_BLOCKED, &gse->flags); }

-static void __clr_tasklet_blocked(struct intel_guc *guc) +static void __clr_tasklet_blocked(struct guc_submit_engine *gse) { - lockdep_assert_held(&guc->sched_engine->lock); - clear_bit(GUC_STATE_TASKLET_BLOCKED, &guc->flags); + lockdep_assert_held(&gse->sched_engine.lock); + clear_bit(GSE_STATE_TASKLET_BLOCKED, &gse->flags); }

-static void clr_tasklet_blocked(struct intel_guc *guc) +static void clr_tasklet_blocked(struct guc_submit_engine *gse) { unsigned long flags;

- spin_lock_irqsave(&guc->sched_engine->lock, flags); - __clr_tasklet_blocked(guc); - spin_unlock_irqrestore(&guc->sched_engine->lock, flags); + spin_lock_irqsave(&gse->sched_engine.lock, flags); + __clr_tasklet_blocked(gse); + spin_unlock_irqrestore(&gse->sched_engine.lock, flags); }

-static bool guc_ids_exhausted(struct intel_guc *guc) +static bool guc_ids_exhausted(struct guc_submit_engine *gse) { - return test_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); + return test_bit(GSE_STATE_GUC_IDS_EXHAUSTED, &gse->flags); }

-static bool test_and_update_guc_ids_exhausted(struct intel_guc *guc) +static bool test_and_update_guc_ids_exhausted(struct guc_submit_engine *gse) { unsigned long flags; bool ret = false; @@ -139,33 +140,33 @@ static bool test_and_update_guc_ids_exhausted(struct intel_guc *guc) * Strict ordering on checking if guc_ids are exhausted isn't required, * so let's avoid grabbing the submission lock if possible. */ - if (guc_ids_exhausted(guc)) { - spin_lock_irqsave(&guc->sched_engine->lock, flags); - ret = guc_ids_exhausted(guc); + if (guc_ids_exhausted(gse)) { + spin_lock_irqsave(&gse->sched_engine.lock, flags); + ret = guc_ids_exhausted(gse); if (ret) - ++guc->total_num_rq_with_no_guc_id; - spin_unlock_irqrestore(&guc->sched_engine->lock, flags); + ++gse->total_num_rq_with_no_guc_id; + spin_unlock_irqrestore(&gse->sched_engine.lock, flags); }

return ret; }

-static void set_and_update_guc_ids_exhausted(struct intel_guc *guc) +static void set_and_update_guc_ids_exhausted(struct guc_submit_engine *gse) { unsigned long flags;

- spin_lock_irqsave(&guc->sched_engine->lock, flags); - ++guc->total_num_rq_with_no_guc_id; - set_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); - spin_unlock_irqrestore(&guc->sched_engine->lock, flags); + spin_lock_irqsave(&gse->sched_engine.lock, flags); + ++gse->total_num_rq_with_no_guc_id; + set_bit(GSE_STATE_GUC_IDS_EXHAUSTED, &gse->flags); + spin_unlock_irqrestore(&gse->sched_engine.lock, flags); }

-static void clr_guc_ids_exhausted(struct intel_guc *guc) +static void clr_guc_ids_exhausted(struct guc_submit_engine *gse) { - lockdep_assert_held(&guc->sched_engine->lock); - GEM_BUG_ON(guc->total_num_rq_with_no_guc_id); + lockdep_assert_held(&gse->sched_engine.lock); + GEM_BUG_ON(gse->total_num_rq_with_no_guc_id);

- clear_bit(GUC_STATE_GUC_IDS_EXHAUSTED, &guc->flags); + clear_bit(GSE_STATE_GUC_IDS_EXHAUSTED, &gse->flags); }

/* @@ -419,6 +420,20 @@ static inline struct intel_guc *ce_to_guc(struct intel_context *ce) return &ce->engine->gt->uc.guc; }

+static inline struct i915_sched_engine * +ce_to_sched_engine(struct intel_context *ce) +{ + return ce->engine->sched_engine; +} + +static inline struct i915_sched_engine * +guc_to_sched_engine(struct intel_guc *guc, int index) +{ + GEM_BUG_ON(index < 0 || index >= GUC_SUBMIT_ENGINE_MAX); + + return &guc->gse[index]->sched_engine; +} + static inline struct i915_priolist *to_priolist(struct rb_node *rb) { return rb_entry(rb, struct i915_priolist, node); @@ -644,19 +659,20 @@ static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) return err; }

-static int guc_add_request(struct intel_guc *guc, struct i915_request *rq) +static int gse_add_request(struct guc_submit_engine *gse, + struct i915_request *rq) { int ret;

- lockdep_assert_held(&guc->sched_engine->lock); + lockdep_assert_held(&gse->sched_engine.lock);

- ret = __guc_add_request(guc, rq); + ret = __guc_add_request(gse->sched_engine.private_data, rq); if (ret == -EBUSY) { - guc->stalled_rq = rq; - guc->submission_stall_reason = STALL_ADD_REQUEST; + gse->stalled_rq = rq; + gse->submission_stall_reason = STALL_ADD_REQUEST; } else { - guc->stalled_rq = NULL; - guc->submission_stall_reason = STALL_NONE; + gse->stalled_rq = NULL; + gse->submission_stall_reason = STALL_NONE; }

return ret; @@ -664,14 +680,15 @@ static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)

static int guc_lrc_desc_pin(struct intel_context *ce, bool loop);

-static int tasklet_register_context(struct intel_guc *guc, +static int tasklet_register_context(struct guc_submit_engine *gse, struct i915_request *rq) { struct intel_context *ce = rq->context; + struct intel_guc *guc = gse->sched_engine.private_data; int ret = 0;

/* Check state */ - lockdep_assert_held(&guc->sched_engine->lock); + lockdep_assert_held(&gse->sched_engine.lock); GEM_BUG_ON(ce->guc_num_rq_submit_no_id); GEM_BUG_ON(request_has_no_guc_id(rq)); GEM_BUG_ON(context_guc_id_invalid(ce)); @@ -694,11 +711,11 @@ static int tasklet_register_context(struct intel_guc *guc, clr_context_needs_register(ce);

if (unlikely(ret == -EBUSY)) { - guc->stalled_rq = rq; - guc->submission_stall_reason = STALL_REGISTER_CONTEXT; + gse->stalled_rq = rq; + gse->submission_stall_reason = STALL_REGISTER_CONTEXT; } else if (unlikely(ret == -EINPROGRESS)) { - guc->stalled_rq = rq; - guc->submission_stall_reason = STALL_DEREGISTER_CONTEXT; + gse->stalled_rq = rq; + gse->submission_stall_reason = STALL_DEREGISTER_CONTEXT; } }

@@ -716,28 +733,29 @@ static inline int rq_prio(const struct i915_request *rq) return rq->sched.attr.priority; }

-static void kick_retire_wq(struct intel_guc *guc) +static void kick_retire_wq(struct guc_submit_engine *gse) { - queue_work(system_unbound_wq, &guc->retire_worker); + queue_work(system_unbound_wq, &gse->retire_worker); }

-static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq); +static int tasklet_pin_guc_id(struct guc_submit_engine *gse, + struct i915_request *rq);

-static int guc_dequeue_one_context(struct intel_guc *guc) +static int gse_dequeue_one_context(struct guc_submit_engine *gse) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; - struct i915_request *last = guc->stalled_rq; + struct i915_sched_engine * const sched_engine = &gse->sched_engine; + struct i915_request *last = gse->stalled_rq; bool submit = !!last; struct rb_node *rb; int ret;

lockdep_assert_held(&sched_engine->lock); - GEM_BUG_ON(guc->stalled_context); - GEM_BUG_ON(!submit && guc->submission_stall_reason); + GEM_BUG_ON(gse->stalled_context); + GEM_BUG_ON(!submit && gse->submission_stall_reason);

if (submit) { /* Flow control conditions */ - switch (guc->submission_stall_reason) { + switch (gse->submission_stall_reason) { case STALL_GUC_ID_TASKLET: goto done; case STALL_REGISTER_CONTEXT: @@ -750,8 +768,8 @@ static int guc_dequeue_one_context(struct intel_guc *guc) GEM_BUG_ON("Invalid stall state"); } } else { - GEM_BUG_ON(!guc->total_num_rq_with_no_guc_id && - guc_ids_exhausted(guc)); + GEM_BUG_ON(!gse->total_num_rq_with_no_guc_id && + guc_ids_exhausted(gse));

while ((rb = rb_first_cached(&sched_engine->queue))) { struct i915_priolist *p = to_priolist(rb); @@ -780,13 +798,13 @@ static int guc_dequeue_one_context(struct intel_guc *guc) struct intel_context *ce = last->context;

if (ce->guc_num_rq_submit_no_id) { - ret = tasklet_pin_guc_id(guc, last); + ret = tasklet_pin_guc_id(gse, last); if (ret) goto blk_tasklet_kick; }

register_context: - ret = tasklet_register_context(guc, last); + ret = tasklet_register_context(gse, last); if (unlikely(ret == -EINPROGRESS)) { goto blk_tasklet; } else if (unlikely(ret == -EPIPE)) { @@ -802,7 +820,7 @@ static int guc_dequeue_one_context(struct intel_guc *guc) guc_set_lrc_tail(last);

add_request: - ret = guc_add_request(guc, last); + ret = gse_add_request(gse, last); if (unlikely(ret == -EPIPE)) { goto deadlk; } else if (ret == -EBUSY) { @@ -817,8 +835,8 @@ static int guc_dequeue_one_context(struct intel_guc *guc) * No requests without a guc_id, enable guc_id allocation at request * creation time (guc_request_alloc). */ - if (!guc->total_num_rq_with_no_guc_id) - clr_guc_ids_exhausted(guc); + if (!gse->total_num_rq_with_no_guc_id) + clr_guc_ids_exhausted(gse);

return submit;

@@ -832,25 +850,26 @@ static int guc_dequeue_one_context(struct intel_guc *guc) return false;

blk_tasklet_kick: - kick_retire_wq(guc); + kick_retire_wq(gse); blk_tasklet: - set_tasklet_blocked(guc); + set_tasklet_blocked(gse); return false; }

-static void guc_submission_tasklet(struct tasklet_struct *t) +static void gse_submission_tasklet(struct tasklet_struct *t) { struct i915_sched_engine *sched_engine = from_tasklet(sched_engine, t, tasklet); - struct intel_guc *guc = sched_engine->private_data; + struct guc_submit_engine *gse = + container_of(sched_engine, typeof(*gse), sched_engine); unsigned long flags; bool loop;

spin_lock_irqsave(&sched_engine->lock, flags);

- if (likely(!tasklet_blocked(guc))) + if (likely(!tasklet_blocked(gse))) do { - loop = guc_dequeue_one_context(guc); + loop = gse_dequeue_one_context(gse); } while (loop);

i915_sched_engine_reset_on_empty(sched_engine); @@ -925,69 +944,99 @@ static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc) } }

-static inline bool -submission_disabled(struct intel_guc *guc) +static bool submission_disabled(struct intel_guc *guc) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; + int i; + + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + struct i915_sched_engine *sched_engine; + + if (unlikely(!guc->gse[i])) + return true; + + sched_engine = guc_to_sched_engine(guc, i); + + if (unlikely(!__tasklet_is_enabled(&sched_engine->tasklet))) + return true; + }

- return unlikely(!sched_engine || - !__tasklet_is_enabled(&sched_engine->tasklet)); + return false; }

-static void kick_tasklet(struct intel_guc *guc) +static void kick_tasklet(struct guc_submit_engine *gse) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; + struct i915_sched_engine *sched_engine = &gse->sched_engine;

- if (likely(!tasklet_blocked(guc))) + if (likely(!tasklet_blocked(gse))) tasklet_hi_schedule(&sched_engine->tasklet); }

static void disable_submission(struct intel_guc *guc) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; + int i;

- if (__tasklet_is_enabled(&sched_engine->tasklet)) { - GEM_BUG_ON(!guc->ct.enabled); - __tasklet_disable_sync_once(&sched_engine->tasklet); - sched_engine->tasklet.callback = NULL; + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + struct i915_sched_engine *sched_engine = + guc_to_sched_engine(guc, i); + + if (__tasklet_is_enabled(&sched_engine->tasklet)) { + GEM_BUG_ON(!guc->ct.enabled); + __tasklet_disable_sync_once(&sched_engine->tasklet); + sched_engine->tasklet.callback = NULL; + } } }

static void enable_submission(struct intel_guc *guc) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; unsigned long flags; + int i;

- spin_lock_irqsave(&guc->sched_engine->lock, flags); - sched_engine->tasklet.callback = guc_submission_tasklet; - wmb(); /* Make sure callback visible */ - if (!__tasklet_is_enabled(&sched_engine->tasklet) && - __tasklet_enable(&sched_engine->tasklet)) { - GEM_BUG_ON(!guc->ct.enabled); - - /* Reset tasklet state */ - guc->stalled_rq = NULL; - if (guc->stalled_context) - intel_context_put(guc->stalled_context); - guc->stalled_context = NULL; - guc->submission_stall_reason = STALL_NONE; - guc->flags = 0; - - /* And kick in case we missed a new request submission. */ - kick_tasklet(guc); + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + struct i915_sched_engine *sched_engine = + guc_to_sched_engine(guc, i); + struct guc_submit_engine *gse = guc->gse[i]; + + spin_lock_irqsave(&sched_engine->lock, flags); + sched_engine->tasklet.callback = gse_submission_tasklet; + wmb(); /* Mask sure callback is visible */ + if (!__tasklet_is_enabled(&sched_engine->tasklet) && + __tasklet_enable(&sched_engine->tasklet)) { + GEM_BUG_ON(!guc->ct.enabled); + + /* Reset GuC submit engine state */ + gse->stalled_rq = NULL; + if (gse->stalled_context) + intel_context_put(gse->stalled_context); + gse->stalled_context = NULL; + gse->submission_stall_reason = STALL_NONE; + gse->flags = 0; + + /* And kick in case we missed a new request submission. */ + kick_tasklet(gse); + } + spin_unlock_irqrestore(&sched_engine->lock, flags); } - spin_unlock_irqrestore(&guc->sched_engine->lock, flags); }

-static void guc_flush_submissions(struct intel_guc *guc) +static void gse_flush_submissions(struct guc_submit_engine *gse) { - struct i915_sched_engine * const sched_engine = guc->sched_engine; + struct i915_sched_engine * const sched_engine = &gse->sched_engine; unsigned long flags;

spin_lock_irqsave(&sched_engine->lock, flags); spin_unlock_irqrestore(&sched_engine->lock, flags); }

+static void guc_flush_submissions(struct intel_guc *guc) +{ + int i; + + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) + if (likely(guc->gse[i])) + gse_flush_submissions(guc->gse[i]); +} + void intel_guc_submission_reset_prepare(struct intel_guc *guc) { int i; @@ -1171,13 +1220,12 @@ void intel_guc_submission_reset(struct intel_guc *guc, bool stalled) if (intel_context_is_pinned(ce)) __guc_reset_context(ce, stalled);

- /* GuC is blown away, drop all references to contexts */ xa_destroy(&guc->context_lookup); }

static void guc_cancel_context_requests(struct intel_context *ce) { - struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine; + struct i915_sched_engine *sched_engine = ce_to_sched_engine(ce); struct i915_request *rq; unsigned long flags;

@@ -1192,8 +1240,9 @@ static void guc_cancel_context_requests(struct intel_context *ce) }

static void -guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine) +gse_cancel_requests(struct guc_submit_engine *gse) { + struct i915_sched_engine *sched_engine = &gse->sched_engine; struct i915_request *rq, *rn; struct rb_node *rb; unsigned long flags; @@ -1250,12 +1299,14 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc) { struct intel_context *ce; unsigned long index; + int i;

xa_for_each(&guc->context_lookup, index, ce) if (intel_context_is_pinned(ce)) guc_cancel_context_requests(ce);

- guc_cancel_sched_engine_requests(guc->sched_engine); + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) + gse_cancel_requests(guc->gse[i]);

/* GuC is blown away, drop all references to contexts */ xa_destroy(&guc->context_lookup); @@ -1283,13 +1334,13 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) intel_gt_unpark_heartbeats(guc_to_gt(guc)); }

-static void retire_worker_sched_disable(struct intel_guc *guc, +static void retire_worker_sched_disable(struct guc_submit_engine *gse, struct intel_context *ce);

static void retire_worker_func(struct work_struct *w) { - struct intel_guc *guc = - container_of(w, struct intel_guc, retire_worker); + struct guc_submit_engine *gse = + container_of(w, struct guc_submit_engine, retire_worker);

/* * It is possible that another thread issues the schedule disable + that @@ -1297,17 +1348,17 @@ static void retire_worker_func(struct work_struct *w) * where nothing needs to be done here. Let's be paranoid and kick the * tasklet in that case. */ - if (guc->submission_stall_reason != STALL_SCHED_DISABLE && - guc->submission_stall_reason != STALL_GUC_ID_WORKQUEUE) { - kick_tasklet(guc); + if (gse->submission_stall_reason != STALL_SCHED_DISABLE && + gse->submission_stall_reason != STALL_GUC_ID_WORKQUEUE) { + kick_tasklet(gse); return; }

- if (guc->submission_stall_reason == STALL_SCHED_DISABLE) { - GEM_BUG_ON(!guc->stalled_context); - GEM_BUG_ON(context_guc_id_invalid(guc->stalled_context)); + if (gse->submission_stall_reason == STALL_SCHED_DISABLE) { + GEM_BUG_ON(!gse->stalled_context); + GEM_BUG_ON(context_guc_id_invalid(gse->stalled_context));

- retire_worker_sched_disable(guc, guc->stalled_context); + retire_worker_sched_disable(gse, gse->stalled_context); }

/* @@ -1315,16 +1366,16 @@ static void retire_worker_func(struct work_struct *w) * albeit after possibly issuing a schedule disable as that is async * operation. */ - intel_gt_retire_requests(guc_to_gt(guc)); + intel_gt_retire_requests(guc_to_gt(gse->sched_engine.private_data));

- if (guc->submission_stall_reason == STALL_GUC_ID_WORKQUEUE) { - GEM_BUG_ON(guc->stalled_context); + if (gse->submission_stall_reason == STALL_GUC_ID_WORKQUEUE) { + GEM_BUG_ON(gse->stalled_context);

/* Hopefully guc_ids are now available, kick tasklet */ - guc->submission_stall_reason = STALL_GUC_ID_TASKLET; - clr_tasklet_blocked(guc); + gse->submission_stall_reason = STALL_GUC_ID_TASKLET; + clr_tasklet_blocked(gse);

- kick_tasklet(guc); + kick_tasklet(gse); } }

@@ -1355,18 +1406,24 @@ int intel_guc_submission_init(struct intel_guc *guc) INIT_LIST_HEAD(&guc->guc_id_list_unpinned); ida_init(&guc->guc_ids);

- INIT_WORK(&guc->retire_worker, retire_worker_func); - return 0; }

void intel_guc_submission_fini(struct intel_guc *guc) { + int i; + if (!guc->lrc_desc_pool) return;

guc_lrc_desc_pool_destroy(guc); - i915_sched_engine_put(guc->sched_engine); + + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + struct i915_sched_engine *sched_engine = + guc_to_sched_engine(guc, i); + + i915_sched_engine_put(sched_engine); + } }

static inline void queue_request(struct i915_sched_engine *sched_engine, @@ -1381,22 +1438,23 @@ static inline void queue_request(struct i915_sched_engine *sched_engine, set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);

if (empty) - kick_tasklet(&rq->engine->gt->uc.guc); + kick_tasklet(ce_to_gse(rq->context)); }

/* Macro to tweak heuristic, using a simple over 50% not ready for now */ #define TOO_MANY_GUC_IDS_NOT_READY(avail, consumed) \ ((consumed) > (avail) / 2) -static bool too_many_guc_ids_not_ready(struct intel_guc *guc, +static bool too_many_guc_ids_not_ready(struct guc_submit_engine *gse, struct intel_context *ce) { u32 available_guc_ids, guc_ids_consumed; + struct intel_guc *guc = gse->sched_engine.private_data;

available_guc_ids = guc->num_guc_ids; - guc_ids_consumed = atomic_read(&guc->num_guc_ids_not_ready); + guc_ids_consumed = atomic_read(&gse->num_guc_ids_not_ready);

if (TOO_MANY_GUC_IDS_NOT_READY(available_guc_ids, guc_ids_consumed)) { - set_and_update_guc_ids_exhausted(guc); + set_and_update_guc_ids_exhausted(gse); return true; }

@@ -1405,34 +1463,36 @@ static bool too_many_guc_ids_not_ready(struct intel_guc *guc,

static void incr_num_rq_not_ready(struct intel_context *ce) { - struct intel_guc *guc = ce_to_guc(ce); + struct guc_submit_engine *gse = ce_to_gse(ce);

if (!atomic_fetch_add(1, &ce->guc_num_rq_not_ready)) - atomic_inc(&guc->num_guc_ids_not_ready); + atomic_inc(&gse->num_guc_ids_not_ready); }

void intel_guc_decr_num_rq_not_ready(struct intel_context *ce) { - struct intel_guc *guc = ce_to_guc(ce); + struct guc_submit_engine *gse = ce_to_gse(ce);

- if (atomic_fetch_add(-1, &ce->guc_num_rq_not_ready) == 1) - atomic_dec(&guc->num_guc_ids_not_ready); + if (atomic_fetch_add(-1, &ce->guc_num_rq_not_ready) == 1) { + GEM_BUG_ON(!atomic_read(&gse->num_guc_ids_not_ready)); + atomic_dec(&gse->num_guc_ids_not_ready); + } }

-static bool need_tasklet(struct intel_guc *guc, struct intel_context *ce) +static bool need_tasklet(struct guc_submit_engine *gse, struct intel_context *ce) { - struct i915_sched_engine * const sched_engine = - ce->engine->sched_engine; + struct i915_sched_engine * const sched_engine = &gse->sched_engine; + struct intel_guc *guc = gse->sched_engine.private_data;

lockdep_assert_held(&sched_engine->lock);

- return guc_ids_exhausted(guc) || submission_disabled(guc) || - guc->stalled_rq || guc->stalled_context || + return guc_ids_exhausted(gse) || submission_disabled(guc) || + gse->stalled_rq || gse->stalled_context || !lrc_desc_registered(guc, ce->guc_id) || !i915_sched_engine_is_empty(sched_engine); }

-static int guc_bypass_tasklet_submit(struct intel_guc *guc, +static int gse_bypass_tasklet_submit(struct guc_submit_engine *gse, struct i915_request *rq) { int ret; @@ -1442,27 +1502,27 @@ static int guc_bypass_tasklet_submit(struct intel_guc *guc, trace_i915_request_in(rq, 0);

guc_set_lrc_tail(rq); - ret = guc_add_request(guc, rq); + ret = gse_add_request(gse, rq);

if (unlikely(ret == -EPIPE)) - disable_submission(guc); + disable_submission(gse->sched_engine.private_data);

return ret; }

static void guc_submit_request(struct i915_request *rq) { - struct i915_sched_engine *sched_engine = rq->engine->sched_engine; - struct intel_guc *guc = &rq->engine->gt->uc.guc; + struct guc_submit_engine *gse = ce_to_gse(rq->context); + struct i915_sched_engine *sched_engine = &gse->sched_engine; unsigned long flags;

/* Will be called from irq-context when using foreign fences. */ spin_lock_irqsave(&sched_engine->lock, flags);

- if (need_tasklet(guc, rq->context)) + if (need_tasklet(gse, rq->context)) queue_request(sched_engine, rq, rq_prio(rq)); - else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY) - kick_tasklet(guc); + else if (gse_bypass_tasklet_submit(gse, rq) == -EBUSY) + kick_tasklet(gse);

spin_unlock_irqrestore(&sched_engine->lock, flags);

@@ -1541,8 +1601,9 @@ static int steal_guc_id(struct intel_guc *guc, bool unpinned) * context. */ if (!unpinned) { - GEM_BUG_ON(guc->stalled_context); - guc->stalled_context = intel_context_get(ce); + GEM_BUG_ON(ce_to_gse(ce)->stalled_context); + + ce_to_gse(ce)->stalled_context = intel_context_get(ce); set_context_guc_id_stolen(ce); } else { set_context_guc_id_invalid(ce); @@ -1604,7 +1665,7 @@ static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce, try_again: spin_lock_irqsave(&guc->contexts_lock, flags);

- if (!tasklet && guc_ids_exhausted(guc)) { + if (!tasklet && guc_ids_exhausted(ce_to_gse(ce))) { ret = -EAGAIN; goto out_unlock; } @@ -2111,7 +2172,7 @@ static void guc_context_ban(struct intel_context *ce, struct i915_request *rq) intel_wakeref_t wakeref; unsigned long flags;

- guc_flush_submissions(guc); + gse_flush_submissions(ce_to_gse(ce));

spin_lock_irqsave(&ce->guc_state.lock, flags); set_context_banned(ce); @@ -2199,7 +2260,7 @@ static void guc_context_sched_disable(struct intel_context *ce) spin_unlock_irqrestore(&ce->guc_state.lock, flags);

with_intel_runtime_pm(runtime_pm, wakeref) - __guc_context_sched_disable(guc, ce, guc_id); + __guc_context_sched_disable(ce_to_guc(ce), ce, guc_id);

return; unpin: @@ -2460,7 +2521,7 @@ static void remove_from_context(struct i915_request *rq) if (likely(!request_has_no_guc_id(rq))) atomic_dec(&ce->guc_id_ref); else - --ce_to_guc(rq->context)->total_num_rq_with_no_guc_id; + --ce_to_gse(rq->context)->total_num_rq_with_no_guc_id; unpin_guc_id(ce_to_guc(ce), ce, false);

i915_request_notify_execute_cb_imm(rq); @@ -2521,13 +2582,14 @@ static void invalidate_guc_id_sched_disable(struct intel_context *ce) clr_context_guc_id_stolen(ce); }

-static void retire_worker_sched_disable(struct intel_guc *guc, +static void retire_worker_sched_disable(struct guc_submit_engine *gse, struct intel_context *ce) { + struct intel_guc *guc = gse->sched_engine.private_data; unsigned long flags; bool disabled;

- guc->stalled_context = NULL; + gse->stalled_context = NULL; spin_lock_irqsave(&ce->guc_state.lock, flags); disabled = submission_disabled(guc); if (!disabled && !context_pending_disable(ce) && context_enabled(ce)) { @@ -2573,10 +2635,10 @@ static void retire_worker_sched_disable(struct intel_guc *guc,

invalidate_guc_id_sched_disable(ce);

- guc->submission_stall_reason = STALL_REGISTER_CONTEXT; - clr_tasklet_blocked(guc); + gse->submission_stall_reason = STALL_REGISTER_CONTEXT; + clr_tasklet_blocked(gse);

- kick_tasklet(ce_to_guc(ce)); + kick_tasklet(gse); }

intel_context_put(ce); @@ -2589,25 +2651,26 @@ static bool context_needs_lrc_desc_pin(struct intel_context *ce, bool new_guc_id !submission_disabled(ce_to_guc(ce)); }

-static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq) +static int tasklet_pin_guc_id(struct guc_submit_engine *gse, + struct i915_request *rq) { struct intel_context *ce = rq->context; int ret = 0;

- lockdep_assert_held(&guc->sched_engine->lock); + lockdep_assert_held(&gse->sched_engine.lock); GEM_BUG_ON(!ce->guc_num_rq_submit_no_id);

if (atomic_add_unless(&ce->guc_id_ref, ce->guc_num_rq_submit_no_id, 0)) goto out;

- ret = pin_guc_id(guc, ce, true); + ret = pin_guc_id(gse->sched_engine.private_data, ce, true); if (unlikely(ret < 0)) { /* * No guc_ids available, disable the tasklet and kick the retire * workqueue hopefully freeing up some guc_ids. */ - guc->stalled_rq = rq; - guc->submission_stall_reason = STALL_GUC_ID_WORKQUEUE; + gse->stalled_rq = rq; + gse->submission_stall_reason = STALL_GUC_ID_WORKQUEUE; return ret; }

@@ -2619,14 +2682,14 @@ static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq) set_context_needs_register(ce);

if (ret == NEW_GUC_ID_ENABLED) { - guc->stalled_rq = rq; - guc->submission_stall_reason = STALL_SCHED_DISABLE; + gse->stalled_rq = rq; + gse->submission_stall_reason = STALL_SCHED_DISABLE; }

clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags); out: - guc->total_num_rq_with_no_guc_id -= ce->guc_num_rq_submit_no_id; - GEM_BUG_ON(guc->total_num_rq_with_no_guc_id < 0); + gse->total_num_rq_with_no_guc_id -= ce->guc_num_rq_submit_no_id; + GEM_BUG_ON(gse->total_num_rq_with_no_guc_id < 0);

list_for_each_entry_reverse(rq, &ce->guc_active.requests, sched.link) if (request_has_no_guc_id(rq)) { @@ -2644,7 +2707,7 @@ static int tasklet_pin_guc_id(struct intel_guc *guc, struct i915_request *rq) * from a context that has scheduling enabled. We have to disable * scheduling before deregistering the context and it isn't safe to do * in the tasklet because of lock inversion (ce->guc_state.lock must be - * acquired before guc->sched_engine->lock). To work around this + * acquired before gse->sched_engine.lock). To work around this * we do the schedule disable in retire workqueue and block the tasklet * until the schedule done G2H returns. Returning non-zero here kicks * the workqueue. @@ -2656,6 +2719,7 @@ static int guc_request_alloc(struct i915_request *rq) { struct intel_context *ce = rq->context; struct intel_guc *guc = ce_to_guc(ce); + struct guc_submit_engine *gse = ce_to_gse(ce); unsigned long flags; int ret;

@@ -2689,8 +2753,8 @@ static int guc_request_alloc(struct i915_request *rq) * ready to submit). Don't allocate one here, defer to submission in the * tasklet. */ - if (test_and_update_guc_ids_exhausted(guc) || - too_many_guc_ids_not_ready(guc, ce)) { + if (test_and_update_guc_ids_exhausted(gse) || + too_many_guc_ids_not_ready(gse, ce)) { set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); goto out; } @@ -2723,7 +2787,7 @@ static int guc_request_alloc(struct i915_request *rq) * submissions we return to allocating guc_ids in this function. */ set_bit(I915_FENCE_FLAG_GUC_ID_NOT_PINNED, &rq->fence.flags); - set_and_update_guc_ids_exhausted(guc); + set_and_update_guc_ids_exhausted(gse); incr_num_rq_not_ready(ce);

return 0; @@ -3131,17 +3195,41 @@ static void guc_sched_engine_destroy(struct kref *kref) { struct i915_sched_engine *sched_engine = container_of(kref, typeof(*sched_engine), ref); - struct intel_guc *guc = sched_engine->private_data; + struct guc_submit_engine *gse = + container_of(sched_engine, typeof(*gse), sched_engine); + struct intel_guc *guc = gse->sched_engine.private_data;

- guc->sched_engine = NULL; + guc->gse[gse->id] = NULL; tasklet_kill(&sched_engine->tasklet); /* flush the callback */ - kfree(sched_engine); + kfree(gse); +} + +static void guc_submit_engine_init(struct intel_guc *guc, + struct guc_submit_engine *gse, + int id) +{ + struct i915_sched_engine *sched_engine = &gse->sched_engine; + + i915_sched_engine_init(sched_engine, ENGINE_VIRTUAL); + INIT_WORK(&gse->retire_worker, retire_worker_func); + tasklet_setup(&sched_engine->tasklet, gse_submission_tasklet); + sched_engine->schedule = i915_schedule; + sched_engine->disabled = guc_sched_engine_disabled; + sched_engine->destroy = guc_sched_engine_destroy; + sched_engine->bump_inflight_request_prio = + guc_bump_inflight_request_prio; + sched_engine->retire_inflight_request_prio = + guc_retire_inflight_request_prio; + sched_engine->private_data = guc; + gse->id = id; }

int intel_guc_submission_setup(struct intel_engine_cs *engine) { struct drm_i915_private *i915 = engine->i915; struct intel_guc *guc = &engine->gt->uc.guc; + struct i915_sched_engine *sched_engine; + int ret, i;

/* * The setup relies on several assumptions (e.g. irqs always enabled) @@ -3149,24 +3237,20 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine) */ GEM_BUG_ON(GRAPHICS_VER(i915) < 11);

- if (!guc->sched_engine) { - guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL); - if (!guc->sched_engine) - return -ENOMEM; - - guc->sched_engine->schedule = i915_schedule; - guc->sched_engine->disabled = guc_sched_engine_disabled; - guc->sched_engine->private_data = guc; - guc->sched_engine->destroy = guc_sched_engine_destroy; - guc->sched_engine->bump_inflight_request_prio = - guc_bump_inflight_request_prio; - guc->sched_engine->retire_inflight_request_prio = - guc_retire_inflight_request_prio; - tasklet_setup(&guc->sched_engine->tasklet, - guc_submission_tasklet); + if (!guc->gse[0]) { + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + guc->gse[i] = kzalloc(sizeof(*guc->gse[i]), GFP_KERNEL); + if (!guc->gse[i]) { + ret = -ENOMEM; + goto put_sched_engine; + } + guc_submit_engine_init(guc, guc->gse[i], i); + } } + + sched_engine = guc_to_sched_engine(guc, GUC_SUBMIT_ENGINE_SINGLE_LRC); i915_sched_engine_put(engine->sched_engine); - engine->sched_engine = i915_sched_engine_get(guc->sched_engine); + engine->sched_engine = i915_sched_engine_get(sched_engine);

guc_default_vfuncs(engine); guc_default_irqs(engine); @@ -3182,6 +3266,16 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine) engine->release = guc_release;

return 0; + +put_sched_engine: + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { + struct i915_sched_engine *sched_engine = + guc_to_sched_engine(guc, i); + + if (sched_engine) + i915_sched_engine_put(sched_engine); + } + return ret; }

void intel_guc_submission_enable(struct intel_guc *guc) @@ -3282,14 +3376,16 @@ int intel_guc_deregister_done_process_msg(struct intel_guc *guc, register_context(ce, true); guc_signal_context_fence(ce); if (context_block_tasklet(ce)) { - GEM_BUG_ON(guc->submission_stall_reason != + struct guc_submit_engine *gse = ce_to_gse(ce); + + GEM_BUG_ON(gse->submission_stall_reason != STALL_DEREGISTER_CONTEXT);

clr_context_block_tasklet(ce); - guc->submission_stall_reason = STALL_MOVE_LRC_TAIL; - clr_tasklet_blocked(guc); + gse->submission_stall_reason = STALL_MOVE_LRC_TAIL; + clr_tasklet_blocked(gse);

- kick_tasklet(ce_to_guc(ce)); + kick_tasklet(gse); } intel_context_put(ce); } else if (context_destroyed(ce)) { @@ -3355,11 +3451,13 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc, spin_unlock_irqrestore(&ce->guc_state.lock, flags);

if (context_block_tasklet(ce)) { + struct guc_submit_engine *gse = ce_to_gse(ce); + clr_context_block_tasklet(ce); - guc->submission_stall_reason = STALL_REGISTER_CONTEXT; - clr_tasklet_blocked(guc); + gse->submission_stall_reason = STALL_REGISTER_CONTEXT; + clr_tasklet_blocked(gse);

- kick_tasklet(ce_to_guc(ce)); + kick_tasklet(gse); }

if (banned) { @@ -3391,7 +3489,7 @@ static void capture_error_state(struct intel_guc *guc, static void guc_context_replay(struct intel_context *ce) { __guc_reset_context(ce, true); - kick_tasklet(ce_to_guc(ce)); + kick_tasklet(ce_to_gse(ce)); }

static void guc_handle_context_reset(struct intel_guc *guc, @@ -3536,35 +3634,32 @@ void intel_guc_dump_active_requests(struct intel_engine_cs *engine, } }

-void intel_guc_submission_print_info(struct intel_guc *guc, - struct drm_printer *p) +static void gse_log_submission_info(struct guc_submit_engine *gse, + struct drm_printer *p, int id) { - struct i915_sched_engine *sched_engine = guc->sched_engine; + struct i915_sched_engine *sched_engine = &gse->sched_engine; struct rb_node *rb; unsigned long flags;

if (!sched_engine) return;

- drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n", - atomic_read(&guc->outstanding_submission_g2h)); - drm_printf(p, "GuC Number GuC IDs: %u\n", guc->num_guc_ids); - drm_printf(p, "GuC Max GuC IDs: %u\n", guc->max_guc_ids); - drm_printf(p, "GuC tasklet count: %u\n", + drm_printf(p, "GSE[%d] tasklet count: %u\n", id, atomic_read(&sched_engine->tasklet.count)); - drm_printf(p, "GuC submit flags: 0x%04lx\n", guc->flags); - drm_printf(p, "GuC total number request without guc_id: %d\n", - guc->total_num_rq_with_no_guc_id); - drm_printf(p, "GuC Number GuC IDs not ready: %d\n", - atomic_read(&guc->num_guc_ids_not_ready)); - drm_printf(p, "GuC stall reason: %d\n", guc->submission_stall_reason); - drm_printf(p, "GuC stalled request: %s\n", - yesno(guc->stalled_rq)); - drm_printf(p, "GuC stalled context: %s\n\n", - yesno(guc->stalled_context)); + drm_printf(p, "GSE[%d] submit flags: 0x%04lx\n", id, gse->flags); + drm_printf(p, "GSE[%d] total number request without guc_id: %d\n", + id, gse->total_num_rq_with_no_guc_id); + drm_printf(p, "GSE[%d] Number GuC IDs not ready: %d\n", + id, atomic_read(&gse->num_guc_ids_not_ready)); + drm_printf(p, "GSE[%d] stall reason: %d\n", + id, gse->submission_stall_reason); + drm_printf(p, "GSE[%d] stalled request: %s\n", + id, yesno(gse->stalled_rq)); + drm_printf(p, "GSE[%d] stalled context: %s\n\n", + id, yesno(gse->stalled_context));

spin_lock_irqsave(&sched_engine->lock, flags); - drm_printf(p, "Requests in GuC submit tasklet:\n"); + drm_printf(p, "Requests in GSE[%d] submit tasklet:\n", id); for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) { struct i915_priolist *pl = to_priolist(rb); struct i915_request *rq; @@ -3594,6 +3689,20 @@ static inline void guc_log_context_priority(struct drm_printer *p, drm_printf(p, "\n"); }

+void intel_guc_submission_print_info(struct intel_guc *guc, + struct drm_printer *p) +{ + int i; + + drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n", + atomic_read(&guc->outstanding_submission_g2h)); + drm_printf(p, "GuC Number GuC IDs: %d\n", guc->num_guc_ids); + drm_printf(p, "GuC Max Number GuC IDs: %d\n\n", guc->max_guc_ids); + + for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) + gse_log_submission_info(guc->gse[i], p, i); +} + void intel_guc_submission_print_context_info(struct intel_guc *guc, struct drm_printer *p) { @@ -3627,6 +3736,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) { struct guc_virtual_engine *ve; struct intel_guc *guc; + struct i915_sched_engine *sched_engine; unsigned int n; int err;

@@ -3635,6 +3745,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) return ERR_PTR(-ENOMEM);

guc = &siblings[0]->gt->uc.guc; + sched_engine = guc_to_sched_engine(guc, GUC_SUBMIT_ENGINE_SINGLE_LRC);

ve->base.i915 = siblings[0]->i915; ve->base.gt = siblings[0]->gt; @@ -3648,7 +3759,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count)

snprintf(ve->base.name, sizeof(ve->base.name), "virtual");

- ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine); + ve->base.sched_engine = i915_sched_engine_get(sched_engine);

ve->base.cops = &virtual_guc_context_ops; ve->base.request_alloc = guc_request_alloc; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h new file mode 100644 index 000000000000..0c224ab18c02 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission_types.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2019 Intel Corporation + */ + +#ifndef _INTEL_GUC_SUBMISSION_TYPES_H_ +#define _INTEL_GUC_SUBMISSION_TYPES_H_ + +#include "gt/intel_engine_types.h" +#include "gt/intel_context_types.h" +#include "i915_scheduler_types.h" + +struct intel_guc; +struct i915_request; + +/* GuC Virtual Engine */ +struct guc_virtual_engine { + struct intel_engine_cs base; + struct intel_context context; +}; + +/* + * Object which encapsulates the globally operated on i915_sched_engine + + * the GuC submission state machine described in intel_guc_submission.c. + */ +struct guc_submit_engine { + struct i915_sched_engine sched_engine; + struct work_struct retire_worker; + struct i915_request *stalled_rq; + struct intel_context *stalled_context; + unsigned long flags; + int total_num_rq_with_no_guc_id; + atomic_t num_guc_ids_not_ready; + int id; + + /* + * Submisson stall reason. See intel_guc_submission.c for detailed + * description. + */ + enum { + STALL_NONE, + STALL_GUC_ID_WORKQUEUE, + STALL_GUC_ID_TASKLET, + STALL_SCHED_DISABLE, + STALL_REGISTER_CONTEXT, + STALL_DEREGISTER_CONTEXT, + STALL_MOVE_LRC_TAIL, + STALL_ADD_REQUEST, + } submission_stall_reason; +}; + +#endif diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index 762127dd56c5..1e7eb49e374c 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -448,15 +448,9 @@ static bool default_disabled(struct i915_sched_engine *sched_engine) return false; }

-struct i915_sched_engine * -i915_sched_engine_create(unsigned int subclass) +void i915_sched_engine_init(struct i915_sched_engine *sched_engine, + unsigned int subclass) { - struct i915_sched_engine *sched_engine; - - sched_engine = kzalloc(sizeof(*sched_engine), GFP_KERNEL); - if (!sched_engine) - return NULL; - kref_init(&sched_engine->ref);

sched_engine->queue = RB_ROOT_CACHED; @@ -481,6 +475,18 @@ i915_sched_engine_create(unsigned int subclass) lock_map_release(&sched_engine->lock.dep_map); local_irq_enable(); #endif +} + +struct i915_sched_engine * +i915_sched_engine_create(unsigned int subclass) +{ + struct i915_sched_engine *sched_engine; + + sched_engine = kzalloc(sizeof(*sched_engine), GFP_KERNEL); + if (!sched_engine) + return NULL; + + i915_sched_engine_init(sched_engine, subclass);

return sched_engine; } diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index 0b9b86af6c7f..4e4ef32b2cbc 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -48,6 +48,9 @@ static inline void i915_priolist_free(struct i915_priolist *p) __i915_priolist_free(p); }

+void i915_sched_engine_init(struct i915_sched_engine *sched_engine, + unsigned int subclass); + struct i915_sched_engine * i915_sched_engine_create(unsigned int subclass);

Dynamically allocate space for lrc descriptor registration with the GuC rather than using a large static buffer indexed by the guc_id. If no space is available to register a context, fall back to tasklet flow control mechanism. Only allow 1/2 of the space to be allocated outside the tasklet to prevent unready requests/contexts from consuming all registration space.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_context_types.h | 3 + drivers/gpu/drm/i915/gt/uc/intel_guc.h | 9 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 202 ++++++++++++------ 3 files changed, 150 insertions(+), 64 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index c01530d7dc67..2df79ba39867 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -179,6 +179,9 @@ struct intel_context { /* GuC scheduling state flags that do not require a lock. */ atomic_t guc_sched_state_no_lock;

+ /* GuC lrc descriptor registration buffer */ + unsigned int guc_lrcd_reg_idx; + /* GuC LRC descriptor ID */ u16 guc_id;

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 8ac016201658..c0a12ae95ba5 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -69,8 +69,13 @@ struct intel_guc { u32 ads_regset_size; u32 ads_golden_ctxt_size;

- struct i915_vma *lrc_desc_pool; - void *lrc_desc_pool_vaddr; + /* GuC LRC descriptor registration */ + struct { + struct i915_vma *vma; + void *vaddr; + struct ida ida; + unsigned int max_idx; + } lrcd_reg;

/* guc_id to intel_context lookup */ struct xarray context_lookup; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index a7f7174b5343..bfda15bf9182 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -439,65 +439,54 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb) return rb_entry(rb, struct i915_priolist, node); }

-static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, u32 index) +static u32 __get_lrc_desc_offset(struct intel_guc *guc, int index) { - struct guc_lrc_desc *base = guc->lrc_desc_pool_vaddr; - + GEM_BUG_ON(index >= guc->lrcd_reg.max_idx); GEM_BUG_ON(index >= guc->max_guc_ids);

- return &base[index]; + return intel_guc_ggtt_offset(guc, guc->lrcd_reg.vma) + + (index * sizeof(struct guc_lrc_desc)); }

-static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id) +static struct guc_lrc_desc *__get_lrc_desc(struct intel_guc *guc, int index) { - struct intel_context *ce = xa_load(&guc->context_lookup, id); + struct guc_lrc_desc *desc;

- GEM_BUG_ON(id >= guc->max_guc_ids); + GEM_BUG_ON(index >= guc->lrcd_reg.max_idx); + GEM_BUG_ON(index >= guc->max_guc_ids);

- return ce; + desc = guc->lrcd_reg.vaddr; + desc = &desc[index]; + memset(desc, 0, sizeof(*desc)); + + return desc; }

-static int guc_lrc_desc_pool_create(struct intel_guc *guc) +static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id) { - u32 size; - int ret; - - size = PAGE_ALIGN(sizeof(struct guc_lrc_desc) * guc->max_guc_ids); - ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool, - (void **)&guc->lrc_desc_pool_vaddr); - if (ret) - return ret; + struct intel_context *ce = xa_load(&guc->context_lookup, id);

- return 0; -} + GEM_BUG_ON(id >= guc->max_guc_ids);

-static void guc_lrc_desc_pool_destroy(struct intel_guc *guc) -{ - guc->lrc_desc_pool_vaddr = NULL; - i915_vma_unpin_and_release(&guc->lrc_desc_pool, I915_VMA_RELEASE_MAP); + return ce; }

static inline bool guc_submission_initialized(struct intel_guc *guc) { - return !!guc->lrc_desc_pool_vaddr; + return !!guc->lrcd_reg.max_idx; }

-static inline void reset_lrc_desc(struct intel_guc *guc, u32 id) +static inline void clr_lrc_desc_registered(struct intel_guc *guc, u32 id) { - if (likely(guc_submission_initialized(guc))) { - struct guc_lrc_desc *desc = __get_lrc_desc(guc, id); - unsigned long flags; - - memset(desc, 0, sizeof(*desc)); + unsigned long flags;

- /* - * xarray API doesn't have xa_erase_irqsave wrapper, so calling - * the lower level functions directly. - */ - xa_lock_irqsave(&guc->context_lookup, flags); - __xa_erase(&guc->context_lookup, id); - xa_unlock_irqrestore(&guc->context_lookup, flags); - } + /* + * xarray API doesn't have xa_erase_irqsave wrapper, so calling + * the lower level functions directly. + */ + xa_lock_irqsave(&guc->context_lookup, flags); + __xa_erase(&guc->context_lookup, id); + xa_unlock_irqrestore(&guc->context_lookup, flags); }

static inline bool lrc_desc_registered(struct intel_guc *guc, u32 id) @@ -1385,6 +1374,9 @@ static void retire_worker_func(struct work_struct *w) } }

+static int guc_lrcd_reg_init(struct intel_guc *guc); +static void guc_lrcd_reg_fini(struct intel_guc *guc); + /* * Set up the memory resources to be shared with the GuC (via the GGTT) * at firmware loading time. @@ -1393,17 +1385,12 @@ int intel_guc_submission_init(struct intel_guc *guc) { int ret;

- if (guc->lrc_desc_pool) + if (guc_submission_initialized(guc)) return 0;

- ret = guc_lrc_desc_pool_create(guc); + ret = guc_lrcd_reg_init(guc); if (ret) return ret; - /* - * Keep static analysers happy, let them know that we allocated the - * vma after testing that it didn't exist earlier. - */ - GEM_BUG_ON(!guc->lrc_desc_pool);

xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);

@@ -1419,10 +1406,10 @@ void intel_guc_submission_fini(struct intel_guc *guc) { int i;

- if (!guc->lrc_desc_pool) + if (!guc_submission_initialized(guc)) return;

- guc_lrc_desc_pool_destroy(guc); + guc_lrcd_reg_fini(guc);

for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) { struct i915_sched_engine *sched_engine = @@ -1495,6 +1482,7 @@ static bool need_tasklet(struct guc_submit_engine *gse, struct intel_context *ce return guc_ids_exhausted(gse) || submission_disabled(guc) || gse->stalled_rq || gse->stalled_context || !lrc_desc_registered(guc, ce->guc_id) || + context_needs_register(ce) || !i915_sched_engine_is_empty(sched_engine); }

@@ -1546,7 +1534,7 @@ static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce) { if (!context_guc_id_invalid(ce)) { ida_simple_remove(&guc->guc_ids, ce->guc_id); - reset_lrc_desc(guc, ce->guc_id); + clr_lrc_desc_registered(guc, ce->guc_id); set_context_guc_id_invalid(ce); } if (!list_empty(&ce->guc_id_link)) @@ -1743,14 +1731,14 @@ static void unpin_guc_id(struct intel_guc *guc, }

static int __guc_action_register_context(struct intel_guc *guc, + struct intel_context *ce, u32 guc_id, - u32 offset, bool loop) { u32 action[] = { INTEL_GUC_ACTION_REGISTER_CONTEXT, guc_id, - offset, + __get_lrc_desc_offset(guc, ce->guc_lrcd_reg_idx), };

return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), @@ -1760,13 +1748,11 @@ static int __guc_action_register_context(struct intel_guc *guc, static int register_context(struct intel_context *ce, bool loop) { struct intel_guc *guc = ce_to_guc(ce); - u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool) + - ce->guc_id * sizeof(struct guc_lrc_desc); int ret;

trace_intel_context_register(ce);

- ret = __guc_action_register_context(guc, ce->guc_id, offset, loop); + ret = __guc_action_register_context(guc, ce, ce->guc_id, loop); if (likely(!ret)) set_context_registered(ce);

@@ -1828,6 +1814,86 @@ static void guc_context_policy_init(struct intel_engine_cs *engine,

static inline u8 map_i915_prio_to_guc_prio(int prio);

+static int alloc_lrcd_reg_idx_buffer(struct intel_guc *guc, int num_per_vma) +{ + u32 size = num_per_vma * sizeof(struct guc_lrc_desc); + struct i915_vma **vma = &guc->lrcd_reg.vma; + void **vaddr = &guc->lrcd_reg.vaddr; + int ret; + + GEM_BUG_ON(!is_power_of_2(size)); + + ret = intel_guc_allocate_and_map_vma(guc, size, vma, vaddr); + if (unlikely(ret)) + return ret; + + guc->lrcd_reg.max_idx += num_per_vma; + + return 0; +} + +static int alloc_lrcd_reg_idx(struct intel_guc *guc, bool tasklet) +{ + int ret; + gfp_t gfp = tasklet ? GFP_ATOMIC : + GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + might_sleep_if(!tasklet); + + /* + * We only allow 1/2 of the space to be allocated outside of tasklet + * (flow control) to ensure requests that are not ready don't consume + * all context registration space. + */ + ret = ida_simple_get(&guc->lrcd_reg.ida, 0, + tasklet ? guc->lrcd_reg.max_idx : + guc->lrcd_reg.max_idx / 2, gfp); + if (unlikely(ret < 0)) + return -EBUSY; + + return ret; +} + +static void __free_lrcd_reg_idx(struct intel_guc *guc, struct intel_context *ce) +{ + if (ce->guc_lrcd_reg_idx && guc->lrcd_reg.max_idx) { + ida_simple_remove(&guc->lrcd_reg.ida, ce->guc_lrcd_reg_idx); + ce->guc_lrcd_reg_idx = 0; + } +} + +static void free_lrcd_reg_idx(struct intel_guc *guc, struct intel_context *ce) +{ + __free_lrcd_reg_idx(guc, ce); +} + +static int guc_lrcd_reg_init(struct intel_guc *guc) +{ + unsigned int buffer_size = I915_GTT_PAGE_SIZE_4K * 16; + int ret; + + ida_init(&guc->lrcd_reg.ida); + + ret = alloc_lrcd_reg_idx_buffer(guc, buffer_size / + sizeof(struct guc_lrc_desc)); + if (unlikely(ret)) + return ret; + + /* Zero is reserved */ + ret = alloc_lrcd_reg_idx(guc, false); + GEM_BUG_ON(ret); + + return ret; +} + +static void guc_lrcd_reg_fini(struct intel_guc *guc) +{ + i915_vma_unpin_and_release(&guc->lrcd_reg.vma, + I915_VMA_RELEASE_MAP); + ida_destroy(&guc->lrcd_reg.ida); + guc->lrcd_reg.max_idx = 0; +} + static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) { struct intel_engine_cs *engine = ce->engine; @@ -1851,6 +1917,14 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) != i915_gem_object_is_lmem(ce->ring->vma->obj));

+ /* Allocate space for registration */ + if (likely(!ce->guc_lrcd_reg_idx)) { + ret = alloc_lrcd_reg_idx(guc, !loop); + if (unlikely(ret < 0)) + return ret; + ce->guc_lrcd_reg_idx = ret; + } + context_registered = lrc_desc_registered(guc, desc_idx);

rcu_read_lock(); @@ -1859,12 +1933,11 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) prio = ctx->sched.priority; rcu_read_unlock();

- reset_lrc_desc(guc, desc_idx); ret = set_lrc_desc_registered(guc, desc_idx, ce); if (unlikely(ret)) return ret;

- desc = __get_lrc_desc(guc, desc_idx); + desc = __get_lrc_desc(guc, ce->guc_lrcd_reg_idx); desc->engine_class = engine_class_to_guc_class(engine->class); desc->engine_submit_mask = adjust_engine_mask(engine->class, engine->mask); @@ -1902,7 +1975,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) } spin_unlock_irqrestore(&ce->guc_state.lock, flags); if (unlikely(disabled)) { - reset_lrc_desc(guc, desc_idx); + clr_lrc_desc_registered(guc, desc_idx); return 0; /* Will get registered later */ } } @@ -1930,7 +2003,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop) with_intel_runtime_pm(runtime_pm, wakeref) ret = register_context(ce, loop); if (unlikely(ret == -EBUSY)) - reset_lrc_desc(guc, desc_idx); + clr_lrc_desc_registered(guc, desc_idx); else if (unlikely(ret == -ENODEV)) ret = 0; /* Will get registered later */ } @@ -2296,6 +2369,7 @@ static void __guc_context_destroy(struct intel_context *ce)

lrc_fini(ce); intel_context_fini(ce); + __free_lrcd_reg_idx(ce_to_guc(ce), ce);

if (intel_engine_is_virtual(ce->engine)) { struct guc_virtual_engine *ve = @@ -2807,11 +2881,11 @@ static int guc_request_alloc(struct i915_request *rq)

if (context_needs_lrc_desc_pin(ce, !!ret)) { ret = guc_lrc_desc_pin(ce, true); - if (unlikely(ret)) { /* unwind */ - if (ret == -EPIPE) { - disable_submission(guc); - goto out; /* GPU will be reset */ - } + if (unlikely(ret == -EBUSY)) + set_context_needs_register(ce); + else if (ret == -EPIPE) + disable_submission(guc); /* GPU will be reset */ + else if (unlikely(ret)) { /* unwind */ atomic_dec(&ce->guc_id_ref); unpin_guc_id(guc, ce, true); return ret; @@ -3438,6 +3512,8 @@ int intel_guc_sched_done_process_msg(struct intel_guc *guc,

if (context_pending_enable(ce)) { clr_context_pending_enable(ce); + + free_lrcd_reg_idx(guc, ce); } else if (context_pending_disable(ce)) { bool banned;

@@ -3706,6 +3782,8 @@ void intel_guc_submission_print_info(struct intel_guc *guc, atomic_read(&guc->outstanding_submission_g2h)); drm_printf(p, "GuC Number GuC IDs: %d\n", guc->num_guc_ids); drm_printf(p, "GuC Max Number GuC IDs: %d\n\n", guc->max_guc_ids); + drm_printf(p, "GuC max context registered: %u\n\n", + guc->lrcd_reg.max_idx);

for (i = 0; i < GUC_SUBMIT_ENGINE_MAX; ++i) gse_log_submission_info(guc->gse[i], p, i);

Sometimes it is desirable to queue work up for later if the GT PM isn't held and run that work on next GT PM unpark.

Implemented with a list in the GT of all pending work, workqueues in the list, a callback to add a workqueue to the list, and finally a wakeref post_get callback that iterates / drains the list + queues the workqueues.

First user of this is deregistration of GuC contexts.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_gt.c | 3 ++ drivers/gpu/drm/i915/gt/intel_gt_pm.c | 8 +++++ .../gpu/drm/i915/gt/intel_gt_pm_unpark_work.c | 35 +++++++++++++++++++ .../gpu/drm/i915/gt/intel_gt_pm_unpark_work.h | 32 +++++++++++++++++ drivers/gpu/drm/i915/gt/intel_gt_types.h | 3 ++ drivers/gpu/drm/i915/gt/uc/intel_guc.h | 3 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 14 +++++--- drivers/gpu/drm/i915/intel_wakeref.c | 5 +++ drivers/gpu/drm/i915/intel_wakeref.h | 1 + 9 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.c create mode 100644 drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.h

diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index a64aa43f7cd9..405558c08d6c 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -29,6 +29,9 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)

spin_lock_init(&gt->irq_lock);

+ spin_lock_init(&gt->pm_unpark_work_lock); + INIT_LIST_HEAD(&gt->pm_unpark_work_list); + INIT_LIST_HEAD(&gt->closed_vma); spin_lock_init(&gt->closed_lock);

diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index dea8e2479897..564c11a3748b 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -90,6 +90,13 @@ static int __gt_unpark(struct intel_wakeref *wf) return 0; }

+static void __gt_unpark_work_queue(struct intel_wakeref *wf) +{ + struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); + + intel_gt_pm_unpark_work_queue(gt); +} + static int __gt_park(struct intel_wakeref *wf) { struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); @@ -118,6 +125,7 @@ static int __gt_park(struct intel_wakeref *wf)

static const struct intel_wakeref_ops wf_ops = { .get = __gt_unpark, + .post_get = __gt_unpark_work_queue, .put = __gt_park, };

diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.c new file mode 100644 index 000000000000..23162dbd0c35 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_runtime_pm.h" +#include "intel_gt_pm.h" + +void intel_gt_pm_unpark_work_queue(struct intel_gt *gt) +{ + struct intel_gt_pm_unpark_work *work, *next; + unsigned long flags; + + spin_lock_irqsave(&gt->pm_unpark_work_lock, flags); + list_for_each_entry_safe(work, next, + &gt->pm_unpark_work_list, link) { + list_del_init(&work->link); + queue_work(system_unbound_wq, &work->worker); + } + spin_unlock_irqrestore(&gt->pm_unpark_work_lock, flags); +} + +void intel_gt_pm_unpark_work_add(struct intel_gt *gt, + struct intel_gt_pm_unpark_work *work) +{ + unsigned long flags; + + spin_lock_irqsave(&gt->pm_unpark_work_lock, flags); + if (intel_gt_pm_is_awake(gt)) + queue_work(system_unbound_wq, &work->worker); + else if (list_empty(&work->link)) + list_add_tail(&work->link, &gt->pm_unpark_work_list); + spin_unlock_irqrestore(&gt->pm_unpark_work_lock, flags); +} diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.h b/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.h new file mode 100644 index 000000000000..08e9011be023 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_unpark_work.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef INTEL_GT_PM_UNPARK_WORK_H +#define INTEL_GT_PM_UNPARK_WORK_H + +#include <linux/list.h> +#include <linux/workqueue.h> + +struct intel_gt; + +struct intel_gt_pm_unpark_work { + struct list_head link; + struct work_struct worker; +}; + +void intel_gt_pm_unpark_work_queue(struct intel_gt *gt); + +void intel_gt_pm_unpark_work_add(struct intel_gt *gt, + struct intel_gt_pm_unpark_work *work); + +static inline void +intel_gt_pm_unpark_work_init(struct intel_gt_pm_unpark_work *work, + work_func_t fn) +{ + INIT_LIST_HEAD(&work->link); + INIT_WORK(&work->worker, fn); +} + +#endif /* INTEL_GT_PM_UNPARK_WORK_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 97a5075288d2..8d8a946561fa 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -91,6 +91,9 @@ struct intel_gt { struct intel_wakeref wakeref; atomic_t user_wakeref;

+ struct list_head pm_unpark_work_list; + spinlock_t pm_unpark_work_lock; /* protect list */ + struct list_head closed_vma; spinlock_t closed_lock; /* guards the list of closed_vma */

diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 72fdfa1f6ccd..aedd5a4281b8 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -18,6 +18,7 @@ #include "intel_uc_fw.h" #include "i915_utils.h" #include "i915_vma.h" +#include "gt/intel_gt_pm_unpark_work.h"

struct __guc_ads_blob;

@@ -63,7 +64,7 @@ struct intel_guc {

spinlock_t destroy_lock; /* protects list / worker */ struct list_head destroyed_contexts; - struct work_struct destroy_worker; + struct intel_gt_pm_unpark_work destroy_worker;

bool submission_supported; bool submission_selected; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 262fa77b56e2..7fe4d1559a81 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1406,8 +1406,9 @@ int intel_guc_submission_init(struct intel_guc *guc) ida_init(&guc->guc_ids);

spin_lock_init(&guc->destroy_lock); + INIT_LIST_HEAD(&guc->destroyed_contexts); - INIT_WORK(&guc->destroy_worker, destroy_worker_func); + intel_gt_pm_unpark_work_init(&guc->destroy_worker, destroy_worker_func);

return 0; } @@ -2446,13 +2447,18 @@ static void deregister_destroyed_contexts(struct intel_guc *guc)

static void destroy_worker_func(struct work_struct *w) { + struct intel_gt_pm_unpark_work *destroy_worker = + container_of(w, struct intel_gt_pm_unpark_work, worker); struct intel_guc *guc = - container_of(w, struct intel_guc, destroy_worker); + container_of(destroy_worker, struct intel_guc, destroy_worker); struct intel_gt *gt = guc_to_gt(guc); int tmp;

- with_intel_gt_pm(gt, tmp) + with_intel_gt_pm_if_awake(gt, tmp) deregister_destroyed_contexts(guc); + + if (!list_empty(&guc->destroyed_contexts)) + intel_gt_pm_unpark_work_add(gt, destroy_worker); }

static void guc_context_destroy(struct kref *kref) @@ -2515,7 +2521,7 @@ static void guc_context_destroy(struct kref *kref) * take the GT PM for the first time which isn't allowed from an atomic * context. */ - queue_work(system_unbound_wq, &guc->destroy_worker); + intel_gt_pm_unpark_work_add(guc_to_gt(guc), &guc->destroy_worker); }

static int guc_context_alloc(struct intel_context *ce) diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c index dfd87d082218..282fc4f312e3 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.c +++ b/drivers/gpu/drm/i915/intel_wakeref.c @@ -24,6 +24,8 @@ static void rpm_put(struct intel_wakeref *wf)

int __intel_wakeref_get_first(struct intel_wakeref *wf) { + bool do_post = false; + /* * Treat get/put as different subclasses, as we may need to run * the put callback from under the shrinker and do not want to @@ -44,8 +46,11 @@ int __intel_wakeref_get_first(struct intel_wakeref *wf) }

smp_mb__before_atomic(); /* release wf->count */ + do_post = true; } atomic_inc(&wf->count); + if (do_post && wf->ops->post_get) + wf->ops->post_get(wf); mutex_unlock(&wf->mutex);

INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index 545c8f277c46..ef7e6a698e8a 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -30,6 +30,7 @@ typedef depot_stack_handle_t intel_wakeref_t;

struct intel_wakeref_ops { int (*get)(struct intel_wakeref *wf); + void (*post_get)(struct intel_wakeref *wf); int (*put)(struct intel_wakeref *wf); };

Calling switch_to_kernel_context isn't needed if the engine PM reference is taken while all contexts are pinned. By not calling switch_to_kernel_context we save on issuing a request to the engine.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_engine_pm.c | 4 ++++ 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 1f07ac4e0672..58099de6bf07 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -162,6 +162,10 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine) unsigned long flags; bool result = true;

+ /* No need to switch_to_kernel_context if GuC submission */ + if (intel_engine_uses_guc(engine)) + return true; + /* GPU is pointing to the void, as good as in the kernel context. */ if (intel_gt_is_wedged(engine->gt)) return true;

Add logical engine mapping. This is required for split-frame, as workloads need to be placed on engines in a logically contiguous manner.

Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/gt/intel_engine_cs.c | 60 ++++++++++++++++--- drivers/gpu/drm/i915/gt/intel_engine_types.h | 1 + .../drm/i915/gt/intel_execlists_submission.c | 1 + drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c | 2 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 21 +------ 5 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 0d9105a31d84..4d790f9a65dd 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -290,7 +290,8 @@ static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) GEM_DEBUG_WARN_ON(iir); }

-static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) +static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, + u8 logical_instance) { const struct engine_info *info = &intel_engines[id]; struct drm_i915_private *i915 = gt->i915; @@ -334,6 +335,7 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id)

engine->class = info->class; engine->instance = info->instance; + engine->logical_mask = BIT(logical_instance); __sprint_engine_name(engine);

engine->props.heartbeat_interval_ms = @@ -572,6 +574,37 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) return info->engine_mask; }

+static void populate_logical_ids(struct intel_gt *gt, u8 *logical_ids, + u8 class, const u8 *map, u8 num_instances) +{ + int i, j; + u8 current_logical_id = 0; + + for (j = 0; j < num_instances; ++j) { + for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) { + if (!HAS_ENGINE(gt, i) || + intel_engines[i].class != class) + continue; + + if (intel_engines[i].instance == map[j]) { + logical_ids[intel_engines[i].instance] = + current_logical_id++; + break; + } + } + } +} + +static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class) +{ + int i; + u8 map[MAX_ENGINE_INSTANCE + 1]; + + for (i = 0; i < MAX_ENGINE_INSTANCE + 1; ++i) + map[i] = i; + populate_logical_ids(gt, logical_ids, class, map, ARRAY_SIZE(map)); +} + /** * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers * @gt: pointer to struct intel_gt @@ -583,7 +616,8 @@ int intel_engines_init_mmio(struct intel_gt *gt) struct drm_i915_private *i915 = gt->i915; const unsigned int engine_mask = init_engine_mask(gt); unsigned int mask = 0; - unsigned int i; + unsigned int i, class; + u8 logical_ids[MAX_ENGINE_INSTANCE + 1]; int err;

drm_WARN_ON(&i915->drm, engine_mask == 0); @@ -593,15 +627,23 @@ int intel_engines_init_mmio(struct intel_gt *gt) if (i915_inject_probe_failure(i915)) return -ENODEV;

- for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { - if (!HAS_ENGINE(gt, i)) - continue; + for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { + setup_logical_ids(gt, logical_ids, class);

- err = intel_engine_setup(gt, i); - if (err) - goto cleanup; + for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) { + u8 instance = intel_engines[i].instance; + + if (intel_engines[i].class != class || + !HAS_ENGINE(gt, i)) + continue;

- mask |= BIT(i); + err = intel_engine_setup(gt, i, + logical_ids[instance]); + if (err) + goto cleanup; + + mask |= BIT(i); + } }

/* diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index ed91bcff20eb..85e5c9a9e502 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -266,6 +266,7 @@ struct intel_engine_cs { unsigned int guc_id;

intel_engine_mask_t mask; + intel_engine_mask_t logical_mask;

u8 class; u8 instance; diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index de5f9c86b9a4..baa1797af1c8 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -3879,6 +3879,7 @@ execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count)

ve->siblings[ve->num_siblings++] = sibling; ve->base.mask |= sibling->mask; + ve->base.logical_mask |= sibling->logical_mask;

/* * All physical engines must be compatible for their emission diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 6926919bcac6..9f5f43a16182 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -176,7 +176,7 @@ static void guc_mapping_table_init(struct intel_gt *gt, for_each_engine(engine, gt, id) { u8 guc_class = engine_class_to_guc_class(engine->class);

- system_info->mapping_table[guc_class][engine->instance] = + system_info->mapping_table[guc_class][ilog2(engine->logical_mask)] = engine->instance; } } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 310116f40509..dec757d319a2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1795,23 +1795,6 @@ static int deregister_context(struct intel_context *ce, u32 guc_id, bool loop) return __guc_action_deregister_context(guc, guc_id, loop); }

-static intel_engine_mask_t adjust_engine_mask(u8 class, intel_engine_mask_t mask) -{ - switch (class) { - case RENDER_CLASS: - return mask >> RCS0; - case VIDEO_ENHANCEMENT_CLASS: - return mask >> VECS0; - case VIDEO_DECODE_CLASS: - return mask >> VCS0; - case COPY_ENGINE_CLASS: - return mask >> BCS0; - default: - MISSING_CASE(class); - return 0; - } -} - static void guc_context_policy_init(struct intel_engine_cs *engine, struct guc_lrc_desc *desc) { @@ -1952,8 +1935,7 @@ static int guc_lrc_desc_pin(struct intel_context *ce, bool loop)

desc = __get_lrc_desc(guc, ce->guc_lrcd_reg_idx); desc->engine_class = engine_class_to_guc_class(engine->class); - desc->engine_submit_mask = adjust_engine_mask(engine->class, - engine->mask); + desc->engine_submit_mask = engine->logical_mask; desc->hw_context_desc = ce->lrc.lrca; ce->guc_prio = map_i915_prio_to_guc_prio(prio); desc->priority = ce->guc_prio; @@ -3978,6 +3960,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count) }

ve->base.mask |= sibling->mask; + ve->base.logical_mask |= sibling->logical_mask;

if (n != 0 && ve->base.class != sibling->class) { DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n",

Expose logical engine instance to user via query engine info IOCTL. This is required for split-frame workloads as these needs to be placed on engines in a logically contiguous order. The logical mapping can change based on fusing. Rather than having user have knowledge of the fusing we simply just expose the logical mapping with the existing query engine info IOCTL.

Cc: Tvrtko Ursulin tvrtko.ursulin@intel.com Signed-off-by: Matthew Brost matthew.brost@intel.com --- drivers/gpu/drm/i915/i915_query.c | 2 ++ include/uapi/drm/i915_drm.h | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index e49da36c62fb..8a72923fbdba 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -124,7 +124,9 @@ query_engine_info(struct drm_i915_private *i915, for_each_uabi_engine(engine, i915) { info.engine.engine_class = engine->uabi_class; info.engine.engine_instance = engine->uabi_instance; + info.flags = I915_ENGINE_INFO_HAS_LOGICAL_INSTANCE; info.capabilities = engine->uabi_capabilities; + info.logical_instance = ilog2(engine->logical_mask);

if (copy_to_user(info_ptr, &info, sizeof(info))) return -EFAULT; diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 7f13d241417f..ef72e07fe08c 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -2706,14 +2706,20 @@ struct drm_i915_engine_info {

/** @flags: Engine flags. */ __u64 flags; +#define I915_ENGINE_INFO_HAS_LOGICAL_INSTANCE (1 << 0)

/** @capabilities: Capabilities of this engine. */ __u64 capabilities; #define I915_VIDEO_CLASS_CAPABILITY_HEVC (1 << 0) #define I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC (1 << 1)

+ /** @logical_instance: Logical instance of engine */ + __u16 logical_instance; + /** @rsvd1: Reserved fields. */ - __u64 rsvd1[4]; + __u16 rsvd1[3]; + /** @rsvd2: Reserved fields. */ + __u64 rsvd2[3]; };

/**