[PATCH 19/21] vdeadline
Chris Wilson
chris at chris-wilson.co.uk
Sat Jun 6 16:01:08 UTC 2020
---
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 13 +-
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 3 +-
drivers/gpu/drm/i915/gt/intel_engine_types.h | 14 +-
drivers/gpu/drm/i915/gt/intel_lrc.c | 293 ++++++---------
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 5 +-
drivers/gpu/drm/i915/gt/selftest_lrc.c | 37 +-
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 6 +-
drivers/gpu/drm/i915/i915_priolist_types.h | 7 +-
drivers/gpu/drm/i915/i915_request.h | 4 +-
drivers/gpu/drm/i915/i915_scheduler.c | 335 +++++++++++++-----
drivers/gpu/drm/i915/i915_scheduler.h | 24 +-
drivers/gpu/drm/i915/i915_scheduler_types.h | 17 +
drivers/gpu/drm/i915/selftests/i915_request.c | 1 +
13 files changed, 454 insertions(+), 305 deletions(-)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index d79307d790da..eb63216f9df8 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -513,7 +513,7 @@ void intel_engine_init_execlists(struct intel_engine_cs *engine)
execlists->active =
memset(execlists->inflight, 0, sizeof(execlists->inflight));
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = I915_DEADLINE_NEVER;
execlists->queue = RB_ROOT_CACHED;
}
@@ -1188,14 +1188,15 @@ bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
}
}
-static int print_sched_attr(const struct i915_sched_attr *attr,
- char *buf, int x, int len)
+static int print_sched(const struct i915_sched_node *node,
+ char *buf, int x, int len)
{
- if (attr->priority == I915_PRIORITY_INVALID)
+ if (node->attr.priority == I915_PRIORITY_INVALID)
return x;
x += snprintf(buf + x, len - x,
- " prio=%d", attr->priority);
+ " prio=%d, dl=%llu",
+ node->attr.priority, node->deadline);
return x;
}
@@ -1208,7 +1209,7 @@ static void print_request(struct drm_printer *m,
char buf[80] = "";
int x = 0;
- x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
+ x = print_sched(&rq->sched, buf, x, sizeof(buf));
drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
prefix,
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index d0a1078ef632..0ba88e38b222 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -188,6 +188,7 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine)
i915_request_add_active_barriers(rq);
/* Install ourselves as a preemption barrier */
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */
/*
@@ -249,7 +250,7 @@ static int __engine_park(struct intel_wakeref *wf)
intel_engine_disarm_breadcrumbs(engine);
/* Must be reset upon idling, or we may miss the busy wakeup. */
- GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
+ GEM_BUG_ON(engine->execlists.queue_deadline_hint != I915_DEADLINE_NEVER);
if (engine->park)
engine->park(engine);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 48e111f16dc5..cc2d95069bf8 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -232,17 +232,7 @@ struct intel_engine_execlists {
unsigned int port_mask;
/**
- * @switch_priority_hint: Second context priority.
- *
- * We submit multiple contexts to the HW simultaneously and would
- * like to occasionally switch between them to emulate timeslicing.
- * To know when timeslicing is suitable, we track the priority of
- * the context submitted second.
- */
- int switch_priority_hint;
-
- /**
- * @queue_priority_hint: Highest pending priority.
+ * @queue_deadline_hint: Next available deadline.
*
* When we add requests into the queue, or adjust the priority of
* executing requests, we compute the maximum priority of those
@@ -253,7 +243,7 @@ struct intel_engine_execlists {
* dequeuing the priority hint may no longer may match the highest
* available request priority.
*/
- int queue_priority_hint;
+ u64 queue_deadline_hint;
/**
* @queue: queue of requests, in priority lists
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 4fea5886929f..a833fce9f04f 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -200,7 +200,7 @@ struct virtual_engine {
*/
struct ve_node {
struct rb_node rb;
- int prio;
+ u64 deadline;
} nodes[I915_NUM_ENGINES];
/*
@@ -414,9 +414,14 @@ static inline int rq_prio(const struct i915_request *rq)
return READ_ONCE(rq->sched.attr.priority);
}
-static int effective_prio(const struct i915_request *rq)
+static inline u64 rq_deadline(const struct i915_request *rq)
{
- int prio = rq_prio(rq);
+ return READ_ONCE(rq->sched.deadline);
+}
+
+static u64 effective_deadline(const struct i915_request *rq)
+{
+ u64 deadline = rq_deadline(rq);
/*
* If this request is special and must not be interrupted at any
@@ -427,27 +432,27 @@ static int effective_prio(const struct i915_request *rq)
* nopreempt for as long as desired).
*/
if (i915_request_has_nopreempt(rq))
- prio = I915_PRIORITY_UNPREEMPTABLE;
+ deadline = 0;
- return prio;
+ return deadline;
}
-static int queue_prio(const struct intel_engine_execlists *execlists)
+static u64 queue_deadline(const struct intel_engine_execlists *execlists)
{
struct rb_node *rb;
rb = rb_first_cached(&execlists->queue);
if (!rb)
- return INT_MIN;
+ return I915_DEADLINE_NEVER;
- return to_priolist(rb)->priority;
+ return to_priolist(rb)->deadline;
}
static inline bool need_preempt(const struct intel_engine_cs *engine,
const struct i915_request *rq,
struct rb_node *rb)
{
- int last_prio;
+ u64 last_deadline;
if (!intel_engine_has_semaphores(engine))
return false;
@@ -470,8 +475,8 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* priority level: the task that is running should remain running
* to preserve FIFO ordering of dependencies.
*/
- last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
- if (engine->execlists.queue_priority_hint <= last_prio)
+ last_deadline = effective_deadline(rq);
+ if (engine->execlists.queue_deadline_hint >= last_deadline)
return false;
/*
@@ -479,7 +484,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* power of PI, be the highest priority of that context.
*/
if (!list_is_last(&rq->sched.link, &engine->active.requests) &&
- rq_prio(list_next_entry(rq, sched.link)) > last_prio)
+ rq_deadline(list_next_entry(rq, sched.link)) < last_deadline)
return true;
if (rb) {
@@ -493,7 +498,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
rcu_read_lock();
next = READ_ONCE(ve->request);
if (next)
- preempt = rq_prio(next) > last_prio;
+ preempt = rq_deadline(next) < last_deadline;
rcu_read_unlock();
}
@@ -511,7 +516,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
* context, it's priority would not exceed ELSP[0] aka last_prio.
*/
- return queue_prio(&engine->execlists) > last_prio;
+ return queue_deadline(&engine->execlists) < last_deadline;
}
__maybe_unused static inline bool
@@ -528,7 +533,7 @@ assert_priority_queue(const struct i915_request *prev,
if (i915_request_is_active(prev))
return true;
- return rq_prio(prev) >= rq_prio(next);
+ return rq_deadline(prev) <= rq_deadline(next);
}
/*
@@ -1098,7 +1103,7 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
{
struct i915_request *rq, *rn, *active = NULL;
struct list_head *uninitialized_var(pl);
- int prio = I915_PRIORITY_INVALID;
+ u64 deadline = I915_DEADLINE_NEVER;
lockdep_assert_held(&engine->active.lock);
@@ -1118,10 +1123,11 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
* engine as load dictates.
*/
if (likely(rq->execution_mask == engine->mask)) {
- GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
- if (rq_prio(rq) != prio) {
- prio = rq_prio(rq);
- pl = i915_sched_lookup_priolist(engine, prio);
+ if (rq_deadline(rq) != deadline) {
+ deadline = rq_deadline(rq);
+ pl = i915_sched_lookup_priolist(engine,
+ deadline);
+
}
GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
@@ -1536,14 +1542,14 @@ dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
if (!rq)
return "";
- snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
+ snprintf(buf, buflen, "%sccid:%x %llx:%lld%s dl %llu",
prefix,
rq->context->lrc.ccid,
rq->fence.context, rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
- rq_prio(rq));
+ rq_deadline(rq));
return buf;
}
@@ -1822,7 +1828,9 @@ static void virtual_xfer_breadcrumbs(struct virtual_engine *ve)
intel_engine_transfer_stale_breadcrumbs(ve->siblings[0], &ve->context);
}
-static void defer_request(struct i915_request *rq, struct list_head * const pl)
+static void defer_request(struct i915_request *rq,
+ struct list_head * const pl,
+ u64 deadline)
{
LIST_HEAD(list);
@@ -1837,6 +1845,7 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
struct i915_dependency *p;
GEM_BUG_ON(i915_request_is_active(rq));
+ rq->sched.deadline = deadline;
list_move_tail(&rq->sched.link, pl);
for_each_waiter(p, rq) {
@@ -1859,10 +1868,9 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
if (!i915_request_is_ready(w))
continue;
- if (rq_prio(w) < rq_prio(rq))
+ if (rq_deadline(w) > deadline)
continue;
- GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
list_move_tail(&w->sched.link, &list);
}
@@ -1873,48 +1881,17 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
static void defer_active(struct intel_engine_cs *engine)
{
struct i915_request *rq;
+ u64 deadline;
rq = __unwind_incomplete_requests(engine);
if (!rq)
return;
- defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq)));
-}
-
-static bool
-need_timeslice(const struct intel_engine_cs *engine,
- const struct i915_request *rq,
- const struct rb_node *rb)
-{
- int hint;
-
- if (!intel_engine_has_timeslices(engine))
- return false;
-
- hint = engine->execlists.queue_priority_hint;
-
- if (rb) {
- const struct virtual_engine *ve =
- rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
- const struct intel_engine_cs *inflight =
- intel_context_inflight(&ve->context);
-
- if (!inflight || inflight == engine) {
- struct i915_request *next;
-
- rcu_read_lock();
- next = READ_ONCE(ve->request);
- if (next)
- hint = max(hint, rq_prio(next));
- rcu_read_unlock();
- }
- }
-
- if (!list_is_last(&rq->sched.link, &engine->active.requests))
- hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
-
- GEM_BUG_ON(hint >= I915_PRIORITY_UNPREEMPTABLE);
- return hint >= effective_prio(rq);
+ deadline = max(rq_deadline(rq),
+ intel_engine_next_virtual_deadline(engine, rq_prio(rq)));
+ defer_request(rq,
+ i915_sched_lookup_priolist(engine, deadline),
+ deadline);
}
static bool
@@ -1937,42 +1914,56 @@ timeslice_yield(const struct intel_engine_execlists *el,
}
static bool
-timeslice_expired(const struct intel_engine_execlists *el,
- const struct i915_request *rq)
+timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
{
- return timer_expired(&el->timer) || timeslice_yield(el, rq);
-}
+ const struct intel_engine_execlists *el = &engine->execlists;
-static int
-switch_prio(struct intel_engine_cs *engine, const struct i915_request *rq)
-{
- if (list_is_last(&rq->sched.link, &engine->active.requests))
- return engine->execlists.queue_priority_hint;
+ if (!intel_engine_has_timeslices(engine))
+ return false;
+
+ if (i915_request_has_nopreempt(rq) && i915_request_started(rq))
+ return false;
+
+ if (timer_expired(&el->timer) || timeslice_yield(el, rq))
+ return true;
- return rq_prio(list_next_entry(rq, sched.link));
+ return rq_deadline(rq) <= i915_sched_to_ticks(ktime_get());
}
-static inline unsigned long
-timeslice(const struct intel_engine_cs *engine)
+static unsigned long timeslice(const struct intel_engine_cs *engine)
{
return READ_ONCE(engine->props.timeslice_duration_ms);
}
-static unsigned long active_timeslice(const struct intel_engine_cs *engine)
+static bool needs_timeslice(const struct intel_engine_cs *engine,
+ const struct i915_request *rq)
{
- const struct intel_engine_execlists *execlists = &engine->execlists;
- const struct i915_request *rq = *execlists->active;
-
+ /* If not currently active, or about to switch, wait for next event */
if (!rq || i915_request_completed(rq))
- return 0;
+ return false;
- if (READ_ONCE(execlists->switch_priority_hint) < effective_prio(rq))
+ /* We do not need to start the timeslice until after the ACK */
+ if (READ_ONCE(engine->execlists.pending[0]))
+ return false;
+
+ /* If ELSP[1] is occupied, always check to see if worth slicing */
+ if (!list_is_last(&rq->sched.link, &engine->active.requests))
+ return true;
+
+ /* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */
+ return READ_ONCE(engine->execlists.queue_deadline_hint) < I915_DEADLINE_NEVER;
+}
+
+static unsigned long active_timeslice(const struct intel_engine_cs *engine)
+{
+ /* Disable the timer if there is nothing to switch to */
+ if (!needs_timeslice(engine, execlists_active(&engine->execlists)))
return 0;
return timeslice(engine);
}
-static void set_timeslice(struct intel_engine_cs *engine)
+static void start_timeslice(struct intel_engine_cs *engine)
{
unsigned long duration;
@@ -1985,29 +1976,6 @@ static void set_timeslice(struct intel_engine_cs *engine)
set_timer_ms(&engine->execlists.timer, duration);
}
-static void start_timeslice(struct intel_engine_cs *engine, int prio)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
- unsigned long duration;
-
- if (!intel_engine_has_timeslices(engine))
- return;
-
- WRITE_ONCE(execlists->switch_priority_hint, prio);
- if (prio == INT_MIN)
- return;
-
- if (timer_pending(&execlists->timer))
- return;
-
- duration = timeslice(engine);
- ENGINE_TRACE(engine,
- "start timeslicing, prio:%d, interval:%lu",
- prio, duration);
-
- set_timer_ms(&execlists->timer, duration);
-}
-
static void record_preemption(struct intel_engine_execlists *execlists)
{
(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
@@ -2123,11 +2091,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
}
ENGINE_TRACE(engine,
- "preempting last=%llx:%lld, prio=%d, hint=%d\n",
+ "preempting last=%llx:%llu, deadline=%llu, hint=%llu\n",
last->fence.context,
last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint);
+ rq_deadline(last),
+ execlists->queue_deadline_hint);
record_preemption(execlists);
/*
@@ -2147,19 +2115,18 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
__unwind_incomplete_requests(engine);
last = NULL;
- } else if (need_timeslice(engine, last, rb) &&
- timeslice_expired(execlists, last)) {
+ } else if (timeslice_expired(engine, last)) {
if (i915_request_completed(last)) {
tasklet_hi_schedule(&execlists->tasklet);
return;
}
ENGINE_TRACE(engine,
- "expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
+ "expired last=%llx:%llu, deadline=%llu, now=%llu, yield?=%s\n",
last->fence.context,
last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint,
+ rq_deadline(last),
+ i915_sched_to_ticks(ktime_get()),
yesno(timeslice_yield(execlists, last)));
ring_set_paused(engine, 1);
@@ -2195,7 +2162,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* Even if ELSP[1] is occupied and not worthy
* of timeslices, our queue might be.
*/
- start_timeslice(engine, queue_prio(execlists));
return;
}
}
@@ -2221,7 +2187,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
GEM_BUG_ON(rq->engine != &ve->base);
GEM_BUG_ON(rq->context != &ve->context);
- if (rq_prio(rq) >= queue_prio(execlists)) {
+ if (rq_deadline(rq) <= queue_deadline(execlists)) {
if (!virtual_matches(ve, rq, engine)) {
spin_unlock(&ve->base.active.lock);
rb = rb_next(rb);
@@ -2230,7 +2196,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (last && !can_merge_rq(last, rq)) {
spin_unlock(&ve->base.active.lock);
- start_timeslice(engine, rq_prio(rq));
return; /* leave this for another sibling */
}
@@ -2244,8 +2209,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
yesno(engine != ve->siblings[0]));
WRITE_ONCE(ve->request, NULL);
- WRITE_ONCE(ve->base.execlists.queue_priority_hint,
- INT_MIN);
+ WRITE_ONCE(ve->base.execlists.queue_deadline_hint,
+ I915_DEADLINE_NEVER);
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
@@ -2399,12 +2364,10 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* request triggering preemption on the next dequeue (or subsequent
* interrupt for secondary ports).
*/
- execlists->queue_priority_hint = queue_prio(execlists);
+ execlists->queue_deadline_hint = queue_deadline(execlists);
if (submit) {
*port = execlists_schedule_in(last, port - execlists->pending);
- execlists->switch_priority_hint =
- switch_prio(engine, *execlists->pending);
/*
* Skip if we ended up with exactly the same set of requests,
@@ -2424,7 +2387,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
set_preempt_timeout(engine, *active);
execlists_submit_ports(engine);
} else {
- start_timeslice(engine, execlists->queue_priority_hint);
skip_submit:
ring_set_paused(engine, 0);
}
@@ -2667,7 +2629,6 @@ static void process_csb(struct intel_engine_cs *engine)
} while (head != tail);
execlists->csb_head = head;
- set_timeslice(engine);
/*
* Gen11 has proven to fail wrt global observation point between
@@ -2816,9 +2777,10 @@ static bool hold_request(const struct i915_request *rq)
return result;
}
-static void __execlists_unhold(struct i915_request *rq)
+static bool __execlists_unhold(struct i915_request *rq)
{
LIST_HEAD(list);
+ bool submit = false;
do {
struct i915_dependency *p;
@@ -2829,10 +2791,7 @@ static void __execlists_unhold(struct i915_request *rq)
GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
i915_request_clear_hold(rq);
- list_move_tail(&rq->sched.link,
- i915_sched_lookup_priolist(rq->engine,
- rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ submit |= intel_engine_queue_request(rq->engine, rq);
/* Also release any children on this engine that are ready */
for_each_waiter(p, rq) {
@@ -2861,6 +2820,8 @@ static void __execlists_unhold(struct i915_request *rq)
rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
} while (rq);
+
+ return submit;
}
static void execlists_unhold(struct intel_engine_cs *engine,
@@ -2872,12 +2833,8 @@ static void execlists_unhold(struct intel_engine_cs *engine,
* Move this request back to the priority queue, and all of its
* children and grandchildren that were suspended along with it.
*/
- __execlists_unhold(rq);
-
- if (rq_prio(rq) > engine->execlists.queue_priority_hint) {
- engine->execlists.queue_priority_hint = rq_prio(rq);
+ if (__execlists_unhold(rq))
tasklet_hi_schedule(&engine->execlists.tasklet);
- }
spin_unlock_irq(&engine->active.lock);
}
@@ -3119,6 +3076,8 @@ static void execlists_submission_tasklet(unsigned long data)
if (unlikely(timeout && preempt_timeout(engine)))
execlists_reset(engine, "preemption time out");
}
+
+ start_timeslice(engine);
}
static void __execlists_kick(struct intel_engine_execlists *execlists)
@@ -3140,15 +3099,6 @@ static void execlists_preempt(struct timer_list *timer)
execlists_kick(timer, preempt);
}
-static void queue_request(struct intel_engine_cs *engine,
- struct i915_request *rq)
-{
- GEM_BUG_ON(!list_empty(&rq->sched.link));
- list_add_tail(&rq->sched.link,
- i915_sched_lookup_priolist(engine, rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
-}
-
static void __submit_queue_imm(struct intel_engine_cs *engine)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -3166,18 +3116,6 @@ static void __submit_queue_imm(struct intel_engine_cs *engine)
__execlists_submission_tasklet(engine);
}
-static void submit_queue(struct intel_engine_cs *engine,
- const struct i915_request *rq)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
-
- if (rq_prio(rq) <= execlists->queue_priority_hint)
- return;
-
- execlists->queue_priority_hint = rq_prio(rq);
- __submit_queue_imm(engine);
-}
-
static bool ancestor_on_hold(const struct intel_engine_cs *engine,
const struct i915_request *rq)
{
@@ -3198,12 +3136,9 @@ static void execlists_submit_request(struct i915_request *request)
list_add_tail(&request->sched.link, &engine->active.hold);
i915_request_set_hold(request);
} else {
- queue_request(engine, request);
-
- GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
- GEM_BUG_ON(list_empty(&request->sched.link));
-
- submit_queue(engine, request);
+ if (intel_engine_queue_request(engine, request))
+ __submit_queue_imm(engine);
+ start_timeslice(engine);
}
spin_unlock_irqrestore(&engine->active.lock, flags);
@@ -4261,7 +4196,7 @@ static void nop_submission_tasklet(unsigned long data)
struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
/* The driver is wedged; don't process any more events. */
- WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN);
+ WRITE_ONCE(engine->execlists.queue_deadline_hint, I915_DEADLINE_NEVER);
}
static void execlists_reset_cancel(struct intel_engine_cs *engine)
@@ -4329,14 +4264,14 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine)
__i915_request_submit(rq);
i915_request_put(rq);
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = I915_DEADLINE_NEVER;
}
spin_unlock(&ve->base.active.lock);
}
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = I915_DEADLINE_NEVER;
execlists->queue = RB_ROOT_CACHED;
GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
@@ -5451,9 +5386,9 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
mask = ve->siblings[0]->mask;
}
- ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
+ ENGINE_TRACE(&ve->base, "rq=%llx:%llu, mask=%x, hint=%llu\n",
rq->fence.context, rq->fence.seqno,
- mask, ve->base.execlists.queue_priority_hint);
+ mask, ve->base.execlists.queue_deadline_hint);
return mask;
}
@@ -5461,7 +5396,7 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
static void virtual_submission_tasklet(unsigned long data)
{
struct virtual_engine * const ve = (struct virtual_engine *)data;
- const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint);
+ const u64 deadline = READ_ONCE(ve->base.execlists.queue_deadline_hint);
intel_engine_mask_t mask;
unsigned int n;
@@ -5501,7 +5436,8 @@ static void virtual_submission_tasklet(unsigned long data)
*/
first = rb_first_cached(&sibling->execlists.virtual) ==
&node->rb;
- if (prio == node->prio || (prio > node->prio && first))
+ if (deadline == node->deadline ||
+ (deadline < node->deadline && first))
goto submit_engine;
rb_erase_cached(&node->rb, &sibling->execlists.virtual);
@@ -5515,7 +5451,7 @@ static void virtual_submission_tasklet(unsigned long data)
rb = *parent;
other = rb_entry(rb, typeof(*other), rb);
- if (prio > other->prio) {
+ if (deadline < other->deadline) {
parent = &rb->rb_left;
} else {
parent = &rb->rb_right;
@@ -5530,8 +5466,8 @@ static void virtual_submission_tasklet(unsigned long data)
submit_engine:
GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
- node->prio = prio;
- if (first && prio > sibling->execlists.queue_priority_hint)
+ node->deadline = deadline;
+ if (first && deadline < sibling->execlists.queue_deadline_hint)
tasklet_hi_schedule(&sibling->execlists.tasklet);
spin_unlock(&sibling->active.lock);
@@ -5563,10 +5499,13 @@ static void virtual_submit_request(struct i915_request *rq)
if (i915_request_completed(rq)) {
__i915_request_submit(rq);
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = I915_DEADLINE_NEVER;
ve->request = NULL;
} else {
- ve->base.execlists.queue_priority_hint = rq_prio(rq);
+ rq->sched.deadline =
+ intel_engine_next_virtual_deadline(&ve->base,
+ rq_prio(rq));
+ ve->base.execlists.queue_deadline_hint = rq_deadline(rq);
ve->request = i915_request_get(rq);
GEM_BUG_ON(!list_empty(virtual_queue(ve)));
@@ -5670,7 +5609,7 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings,
ve->base.bond_execute = virtual_bond_execute;
INIT_LIST_HEAD(virtual_queue(ve));
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = I915_DEADLINE_NEVER;
tasklet_init(&ve->base.execlists.tasklet,
virtual_submission_tasklet,
(unsigned long)ve);
@@ -5857,12 +5796,10 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine,
show_request(m, last, "\t\tE ");
}
- if (execlists->switch_priority_hint != INT_MIN)
- drm_printf(m, "\t\tSwitch priority hint: %d\n",
- READ_ONCE(execlists->switch_priority_hint));
- if (execlists->queue_priority_hint != INT_MIN)
- drm_printf(m, "\t\tQueue priority hint: %d\n",
- READ_ONCE(execlists->queue_priority_hint));
+ if (execlists->queue_deadline_hint != I915_DEADLINE_NEVER)
+ drm_printf(m, "\t\tQueue deadline hint: %llu, now: %llu\n",
+ READ_ONCE(execlists->queue_deadline_hint),
+ i915_sched_to_ticks(ktime_get()));
last = NULL;
count = 0;
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index afa4f88035ac..01fca8acd4c4 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -879,7 +879,10 @@ static int __igt_reset_engines(struct intel_gt *gt,
break;
}
- if (i915_request_wait(rq, 0, HZ / 5) < 0) {
+ /* With deadlines, no strict priority */
+ i915_request_set_deadline(rq, 0);
+
+ if (i915_request_wait(rq, 0, HZ / 2) < 0) {
struct drm_printer p =
drm_info_printer(gt->i915->drm.dev);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 5cfadab091ea..f0fe3c62c6ec 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -85,6 +85,9 @@ static int wait_for_submit(struct intel_engine_cs *engine,
struct i915_request *rq,
unsigned long timeout)
{
+ /* Ignore our own attempts to suppress excess tasklets */
+ tasklet_hi_schedule(&engine->execlists.tasklet);
+
timeout += jiffies;
do {
bool done = time_after(jiffies, timeout);
@@ -754,7 +757,7 @@ semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
static int
release_queue(struct intel_engine_cs *engine,
struct i915_vma *vma,
- int idx, int prio)
+ int idx, u64 deadline)
{
struct i915_request *rq;
u32 *cs;
@@ -779,10 +782,7 @@ release_queue(struct intel_engine_cs *engine,
i915_request_get(rq);
i915_request_add(rq);
- local_bh_disable();
- i915_request_set_priority(rq, prio);
- local_bh_enable(); /* kick tasklet */
-
+ i915_request_set_deadline(rq, deadline);
i915_request_put(rq);
return 0;
@@ -796,6 +796,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
struct intel_engine_cs *engine;
struct i915_request *head;
enum intel_engine_id id;
+ long timeout;
int err, i, n = 0;
head = semaphore_queue(outer, vma, n++);
@@ -816,12 +817,16 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
}
}
- err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
+ err = release_queue(outer, vma, n, 0);
if (err)
goto out;
- if (i915_request_wait(head, 0,
- 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
+ /* Expected number of pessimal slices required */
+ timeout = RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3);
+ timeout *= 4; /* safety factor, including bucketing */
+ timeout += HZ / 2; /* and include the request completion */
+
+ if (i915_request_wait(head, 0, timeout) < 0) {
pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
count, n);
GEM_TRACE_DUMP();
@@ -930,6 +935,8 @@ create_rewinder(struct intel_context *ce,
err = i915_request_await_dma_fence(rq, &wait->fence);
if (err)
goto err;
+
+ i915_request_set_deadline(rq, rq_deadline(wait));
}
cs = intel_ring_begin(rq, 14);
@@ -1204,7 +1211,7 @@ static int live_timeslice_queue(void *arg)
err = PTR_ERR(rq);
goto err_heartbeat;
}
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
err = wait_for_submit(engine, rq, HZ / 2);
if (err) {
pr_err("%s: Timed out trying to submit semaphores\n",
@@ -1227,10 +1234,9 @@ static int live_timeslice_queue(void *arg)
}
GEM_BUG_ON(i915_request_completed(rq));
- GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
/* Queue: semaphore signal, matching priority as semaphore */
- err = release_queue(engine, vma, 1, effective_prio(rq));
+ err = release_queue(engine, vma, 1, effective_deadline(rq));
if (err)
goto err_rq;
@@ -1357,6 +1363,7 @@ static int live_timeslice_nopreempt(void *arg)
goto out_spin;
}
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
i915_request_get(rq);
i915_request_add(rq);
@@ -1729,6 +1736,7 @@ static int live_late_preempt(void *arg)
/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
ctx_lo->sched.priority = 1;
+ ctx_hi->sched.priority = I915_PRIORITY_MIN;
for_each_engine(engine, gt, id) {
struct igt_live_test t;
@@ -2852,6 +2860,9 @@ static int live_preempt_gang(void *arg)
struct i915_request *n =
list_next_entry(rq, client_link);
+ /* With deadlines, no strict priority ordering */
+ i915_request_set_deadline(rq, 0);
+
if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
struct drm_printer p =
drm_info_printer(engine->i915->drm.dev);
@@ -3073,7 +3084,7 @@ static int preempt_user(struct intel_engine_cs *engine,
i915_request_get(rq);
i915_request_add(rq);
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
if (i915_request_wait(rq, 0, HZ / 2) < 0)
err = -ETIME;
@@ -4607,6 +4618,7 @@ static int emit_semaphore_signal(struct intel_context *ce, void *slot)
intel_ring_advance(rq, cs);
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
i915_request_add(rq);
return 0;
@@ -5673,6 +5685,7 @@ static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
intel_ring_advance(rq, cs);
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
err_rq:
i915_request_add(rq);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 0c42e8b0c211..847248e017e9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -333,8 +333,8 @@ static void __guc_dequeue(struct intel_engine_cs *engine)
i915_priolist_free(p);
}
done:
- execlists->queue_priority_hint =
- rb ? to_priolist(rb)->priority : INT_MIN;
+ execlists->queue_deadline_hint =
+ rb ? to_priolist(rb)->deadline : I915_DEADLINE_NEVER;
if (submit) {
*port = schedule_in(last, port - execlists->inflight);
*++port = NULL;
@@ -476,7 +476,7 @@ static void guc_reset_cancel(struct intel_engine_cs *engine)
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = I915_DEADLINE_NEVER;
execlists->queue = RB_ROOT_CACHED;
spin_unlock_irqrestore(&engine->active.lock, flags);
diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h
index bc2fa84f98a8..43a0ac45295f 100644
--- a/drivers/gpu/drm/i915/i915_priolist_types.h
+++ b/drivers/gpu/drm/i915/i915_priolist_types.h
@@ -22,6 +22,8 @@ enum {
/* Interactive workload, scheduled for immediate pageflipping */
I915_PRIORITY_DISPLAY,
+
+ __I915_PRIORITY_KERNEL__
};
/* Smallest priority value that cannot be bumped. */
@@ -35,13 +37,12 @@ enum {
* i.e. nothing can have higher priority and force us to usurp the
* active request.
*/
-#define I915_PRIORITY_UNPREEMPTABLE INT_MAX
-#define I915_PRIORITY_BARRIER (I915_PRIORITY_UNPREEMPTABLE - 1)
+#define I915_PRIORITY_BARRIER INT_MAX
struct i915_priolist {
struct list_head requests;
struct rb_node node;
- int priority;
+ u64 deadline;
};
#endif /* _I915_PRIOLIST_TYPES_H_ */
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 118ab6650d1f..23594e712292 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -561,7 +561,7 @@ static inline void i915_request_clear_hold(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_timeline(struct i915_request *rq)
+i915_request_timeline(const struct i915_request *rq)
{
/* Valid only while the request is being constructed (or retired). */
return rcu_dereference_protected(rq->timeline,
@@ -576,7 +576,7 @@ i915_request_gem_context(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_active_timeline(struct i915_request *rq)
+i915_request_active_timeline(const struct i915_request *rq)
{
/*
* When in use during submission, we are protected by a guarantee that
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index 0f819ad228fe..1babed052e78 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -20,6 +20,11 @@ static struct i915_global_scheduler {
static DEFINE_SPINLOCK(ipi_lock);
static LIST_HEAD(ipi_list);
+static inline u64 rq_deadline(const struct i915_request *rq)
+{
+ return READ_ONCE(rq->sched.deadline);
+}
+
static inline int rq_prio(const struct i915_request *rq)
{
return READ_ONCE(rq->sched.attr.priority);
@@ -32,6 +37,7 @@ static void ipi_schedule(struct irq_work *wrk)
struct i915_dependency *p;
struct i915_request *rq;
unsigned long flags;
+ u64 deadline;
int prio;
spin_lock_irqsave(&ipi_lock, flags);
@@ -40,7 +46,10 @@ static void ipi_schedule(struct irq_work *wrk)
rq = container_of(p->signaler, typeof(*rq), sched);
list_del_init(&p->ipi_link);
+ deadline = p->ipi_deadline;
prio = p->ipi_priority;
+
+ p->ipi_deadline = I915_DEADLINE_NEVER;
p->ipi_priority = I915_PRIORITY_INVALID;
}
spin_unlock_irqrestore(&ipi_lock, flags);
@@ -49,6 +58,8 @@ static void ipi_schedule(struct irq_work *wrk)
if (prio > rq_prio(rq) && !i915_request_completed(rq))
i915_request_set_priority(rq, prio);
+ if (deadline < rq_deadline(rq) && !i915_request_completed(rq))
+ i915_request_set_deadline(rq, deadline);
} while (1);
rcu_read_unlock();
}
@@ -76,28 +87,8 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb)
return rb_entry(rb, struct i915_priolist, node);
}
-static void assert_priolists(struct intel_engine_execlists * const execlists)
-{
- struct rb_node *rb;
- long last_prio;
-
- if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
- return;
-
- GEM_BUG_ON(rb_first_cached(&execlists->queue) !=
- rb_first(&execlists->queue.rb_root));
-
- last_prio = INT_MAX;
- for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
- const struct i915_priolist *p = to_priolist(rb);
-
- GEM_BUG_ON(p->priority > last_prio);
- last_prio = p->priority;
- }
-}
-
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
struct i915_priolist *p;
@@ -105,10 +96,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
bool first = true;
lockdep_assert_held(&engine->active.lock);
- assert_priolists(execlists);
if (unlikely(execlists->no_priolist))
- prio = I915_PRIORITY_NORMAL;
+ deadline = 0;
find_priolist:
/* most positive priority is scheduled first, equal priorities fifo */
@@ -117,9 +107,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
while (*parent) {
rb = *parent;
p = to_priolist(rb);
- if (prio > p->priority) {
+ if (deadline < p->deadline) {
parent = &rb->rb_left;
- } else if (prio < p->priority) {
+ } else if (deadline > p->deadline) {
parent = &rb->rb_right;
first = false;
} else {
@@ -127,13 +117,13 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- if (prio == I915_PRIORITY_NORMAL) {
+ if (!deadline) {
p = &execlists->default_priolist;
} else {
p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC);
/* Convert an allocation failure to a priority bump */
if (unlikely(!p)) {
- prio = I915_PRIORITY_NORMAL; /* recurses just once */
+ deadline = 0; /* recurses just once */
/* To maintain ordering with all rendering, after an
* allocation failure we have to disable all scheduling.
@@ -148,7 +138,7 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- p->priority = prio;
+ p->deadline = deadline;
INIT_LIST_HEAD(&p->requests);
rb_link_node(&p->node, rb, parent);
@@ -157,70 +147,252 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
return &p->requests;
}
-void __i915_priolist_free(struct i915_priolist *p)
-{
- kmem_cache_free(global.slab_priorities, p);
-}
-
-static inline bool need_preempt(int prio, int active)
+void i915_priolist_free(struct i915_priolist *p)
{
- /*
- * Allow preemption of low -> normal -> high, but we do
- * not allow low priority tasks to preempt other low priority
- * tasks under the impression that latency for low priority
- * tasks does not matter (as much as background throughput),
- * so kiss.
- */
- return prio >= max(I915_PRIORITY_NORMAL, active);
+ if (p->deadline)
+ kmem_cache_free(global.slab_priorities, p);
}
-static void kick_submission(struct intel_engine_cs *engine,
+static bool kick_submission(struct intel_engine_cs *engine,
const struct i915_request *rq,
- int prio)
+ u64 deadline)
{
const struct i915_request *inflight;
-
- /*
- * We only need to kick the tasklet once for the high priority
- * new context we add into the queue.
- */
- if (prio <= engine->execlists.queue_priority_hint)
- return;
-
- rcu_read_lock();
+ u64 preempt = I915_DEADLINE_NEVER;
/* Nothing currently active? We're overdue for a submission! */
inflight = execlists_active(&engine->execlists);
if (!inflight)
- goto unlock;
+ goto out;
/*
* If we are already the currently executing context, don't
* bother evaluating if we should preempt ourselves.
*/
if (inflight->context == rq->context)
- goto unlock;
+ return false;
+ preempt = rq_deadline(inflight);
ENGINE_TRACE(engine,
- "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
- prio,
+ "bumping queue-deadline-hint:%llu for rq:%llx:%lld, inflight:%llx:%lld deadline %llu\n",
+ deadline,
rq->fence.context, rq->fence.seqno,
inflight->fence.context, inflight->fence.seqno,
- inflight->sched.attr.priority);
+ preempt);
+
+ /*
+ * We only need to kick the tasklet once for the high priority
+ * new context we add into the queue.
+ */
+out:
+ engine->execlists.queue_deadline_hint = deadline;
+ return deadline < preempt;
+}
+
+static bool __i915_request_set_deadline(struct i915_request *rq, u64 deadline)
+{
+ struct intel_engine_cs *engine = rq->engine;
+ struct i915_request *rn;
+ struct list_head *plist;
+ bool kick = false;
+ LIST_HEAD(dfs);
+
+ lockdep_assert_held(&engine->active.lock);
+ list_add(&rq->sched.dfs, &dfs);
+
+ list_for_each_entry(rq, &dfs, sched.dfs) {
+ struct i915_dependency *p;
+
+ GEM_BUG_ON(rq->engine != engine);
+
+ for_each_signaler(p, rq) {
+ struct i915_request *s =
+ container_of(p->signaler, typeof(*s), sched);
+
+ GEM_BUG_ON(s == rq);
+
+ if (rq_deadline(s) <= deadline)
+ continue;
+
+ if (i915_request_completed(s))
+ continue;
+
+ if (s->engine != rq->engine) {
+ spin_lock(&ipi_lock);
+ if (deadline < p->ipi_deadline) {
+ p->ipi_deadline = deadline;
+ list_move(&p->ipi_link, &ipi_list);
+ irq_work_queue(&ipi_work);
+ }
+ spin_unlock(&ipi_lock);
+ continue;
+ }
+
+ list_move_tail(&s->sched.dfs, &dfs);
+ }
+ }
+
+ plist = i915_sched_lookup_priolist(engine, deadline);
+
+ /* Fifo and depth-first replacement ensure our deps execute first */
+ list_for_each_entry_safe_reverse(rq, rn, &dfs, sched.dfs) {
+ GEM_BUG_ON(rq->engine != engine);
+ GEM_BUG_ON(deadline > rq_deadline(rq));
+
+ INIT_LIST_HEAD(&rq->sched.dfs);
+ WRITE_ONCE(rq->sched.deadline, deadline);
+ RQ_TRACE(rq, "set-deadline:%llu\n", deadline);
+
+ /*
+ * Once the request is ready, it will be placed into the
+ * priority lists and then onto the HW runlist. Before the
+ * request is ready, it does not contribute to our preemption
+ * decisions and we can safely ignore it, as it will, and
+ * any preemption required, be dealt with upon submission.
+ * See engine->submit_request()
+ */
+
+ if (i915_request_in_priority_queue(rq))
+ list_move_tail(&rq->sched.link, plist);
+
+ /* Defer (tasklet) submission until after all updates. */
+ if (i915_request_is_ready(rq) &&
+ deadline < engine->execlists.queue_deadline_hint) {
+ rcu_read_lock();
+ kick |= kick_submission(engine, rq, deadline);
+ rcu_read_unlock();
+ }
+ }
+
+ return kick;
+}
+
+void i915_request_set_deadline(struct i915_request *rq, u64 deadline)
+{
+ struct intel_engine_cs *engine = READ_ONCE(rq->engine);
+ unsigned long flags;
+
+ if (!intel_engine_has_scheduler(engine))
+ return;
- engine->execlists.queue_priority_hint = prio;
- if (need_preempt(prio, rq_prio(inflight)))
+ /*
+ * Virtual engines complicate acquiring the engine timeline lock,
+ * as their rq->engine pointer is not stable until under that
+ * engine lock. The simple ploy we use is to take the lock then
+ * check that the rq still belongs to the newly locked engine.
+ */
+ spin_lock_irqsave(&engine->active.lock, flags);
+ while (engine != READ_ONCE(rq->engine)) {
+ spin_unlock(&engine->active.lock);
+ engine = READ_ONCE(rq->engine);
+ spin_lock(&engine->active.lock);
+ }
+
+ if (i915_request_completed(rq))
+ goto unlock;
+
+ if (deadline >= rq_deadline(rq))
+ goto unlock;
+
+ if (__i915_request_set_deadline(rq, deadline))
tasklet_hi_schedule(&engine->execlists.tasklet);
unlock:
+ spin_unlock_irqrestore(&engine->active.lock, flags);
+}
+
+static u64 prio_slice(const struct intel_engine_cs *engine, int prio)
+{
+ u64 slice;
+ int sf;
+
+ /*
+ * With a 1ms scheduling quantum:
+ *
+ * MAX USER: ~1ms deadline
+ * 0: ~16ms deadline
+ * MIN_USER: 1000ms deadline
+ */
+
+ if (prio >= __I915_PRIORITY_KERNEL__)
+ return INT_MAX - prio;
+
+ slice = __I915_PRIORITY_KERNEL__ - prio;
+ if (prio >= 0)
+ sf = 20 - 6;
+ else
+ sf = 20 - 1;
+
+ return slice << sf;
+}
+
+u64 intel_engine_virtual_deadline(const struct intel_engine_cs *engine,
+ u64 kt, int priority)
+{
+ return i915_sched_to_ticks(kt + prio_slice(engine, priority));
+}
+
+u64 intel_engine_next_virtual_deadline(const struct intel_engine_cs *engine,
+ int priority)
+{
+ return intel_engine_virtual_deadline(engine, ktime_get(), priority);
+}
+
+static u64 signal_deadline(const struct i915_request *rq)
+{
+ u64 last = ktime_to_ns(ktime_get());
+ const struct i915_dependency *p;
+
+ /*
+ * Find the earliest point at which we will become 'ready',
+ * which we infer from the deadline of all active signalers.
+ * We will position ourselves at the end of that chain of work.
+ */
+
+ rcu_read_lock();
+ for_each_signaler(p, rq) {
+ const struct i915_request *s =
+ container_of(p->signaler, typeof(*s), sched);
+
+ if (i915_request_completed(s))
+ continue;
+
+ if (rq_prio(s) < rq_prio(rq))
+ continue;
+
+ last = max(last, i915_sched_to_ns(rq_deadline(s)));
+ }
rcu_read_unlock();
+
+ return last;
+}
+
+static u64 earliest_deadline(const struct intel_engine_cs *engine,
+ const struct i915_request *rq)
+{
+ return intel_engine_virtual_deadline(engine,
+ signal_deadline(rq),
+ rq_prio(rq));
+}
+
+static bool set_earliest_deadline(const struct intel_engine_cs *engine,
+ struct i915_request *rq, u64 old)
+{
+ u64 dl;
+
+ /* Recompute our deadlines and promote after a priority change */
+ dl = min(earliest_deadline(engine, rq), rq_deadline(rq));
+ if (dl >= old)
+ return false;
+
+ return __i915_request_set_deadline(rq, dl);
}
-static void __i915_request_set_priority(struct i915_request *rq, int prio)
+static bool __i915_request_set_priority(struct i915_request *rq, int prio)
{
struct intel_engine_cs *engine = rq->engine;
struct i915_request *rn;
- struct list_head *plist;
+ bool kick = false;
LIST_HEAD(dfs);
lockdep_assert_held(&engine->active.lock);
@@ -277,32 +449,20 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
}
}
- plist = i915_sched_lookup_priolist(engine, prio);
-
- /* Fifo and depth-first replacement ensure our deps execute first */
list_for_each_entry_safe_reverse(rq, rn, &dfs, sched.dfs) {
GEM_BUG_ON(rq->engine != engine);
+ GEM_BUG_ON(prio < rq_prio(rq));
INIT_LIST_HEAD(&rq->sched.dfs);
WRITE_ONCE(rq->sched.attr.priority, prio);
+ RQ_TRACE(rq, "set-priority:%d\n", prio);
- /*
- * Once the request is ready, it will be placed into the
- * priority lists and then onto the HW runlist. Before the
- * request is ready, it does not contribute to our preemption
- * decisions and we can safely ignore it, as it will, and
- * any preemption required, be dealt with upon submission.
- * See engine->submit_request()
- */
- if (!i915_request_is_ready(rq))
- continue;
-
- if (i915_request_in_priority_queue(rq))
- list_move_tail(&rq->sched.link, plist);
-
- /* Defer (tasklet) submission until after all updates. */
- kick_submission(engine, rq, prio);
+ if (i915_request_is_ready(rq) &&
+ set_earliest_deadline(engine, rq, rq_deadline(rq)))
+ kick = true;
}
+
+ return kick;
}
void i915_request_set_priority(struct i915_request *rq, int prio)
@@ -332,12 +492,21 @@ void i915_request_set_priority(struct i915_request *rq, int prio)
if (prio <= rq_prio(rq))
goto unlock;
- __i915_request_set_priority(rq, prio);
+ if (__i915_request_set_priority(rq, prio))
+ tasklet_hi_schedule(&engine->execlists.tasklet);
unlock:
spin_unlock_irqrestore(&engine->active.lock, flags);
}
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq)
+{
+ lockdep_assert_held(&engine->active.lock);
+ set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ return set_earliest_deadline(engine, rq, I915_DEADLINE_NEVER);
+}
+
void i915_sched_node_init(struct i915_sched_node *node)
{
spin_lock_init(&node->lock);
@@ -353,6 +522,7 @@ void i915_sched_node_init(struct i915_sched_node *node)
void i915_sched_node_reinit(struct i915_sched_node *node)
{
node->attr.priority = I915_PRIORITY_INVALID;
+ node->deadline = I915_DEADLINE_NEVER;
node->semaphores = 0;
node->flags = 0;
@@ -385,6 +555,7 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
if (!node_signaled(signal)) {
INIT_LIST_HEAD(&dep->ipi_link);
+ dep->ipi_deadline = I915_DEADLINE_NEVER;
dep->ipi_priority = I915_PRIORITY_INVALID;
dep->signaler = signal;
dep->waiter = node;
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index b26a13ef6feb..c546f6d66136 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -37,15 +37,29 @@ int i915_sched_node_add_dependency(struct i915_sched_node *node,
void i915_sched_node_retire(struct i915_sched_node *node);
void i915_request_set_priority(struct i915_request *request, int prio);
+void i915_request_set_deadline(struct i915_request *request, u64 deadline);
+
+u64 intel_engine_virtual_deadline(const struct intel_engine_cs *engine,
+ u64 kt, int priority);
+u64 intel_engine_next_virtual_deadline(const struct intel_engine_cs *engine,
+ int priority);
+
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq);
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio);
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline);
+
+void i915_priolist_free(struct i915_priolist *p);
+
+static inline u64 i915_sched_to_ticks(ktime_t kt)
+{
+ return ktime_to_ns(kt) >> I915_SCHED_DEADLINE_SHIFT;
+}
-void __i915_priolist_free(struct i915_priolist *p);
-static inline void i915_priolist_free(struct i915_priolist *p)
+static inline u64 i915_sched_to_ns(u64 deadline)
{
- if (p->priority != I915_PRIORITY_NORMAL)
- __i915_priolist_free(p);
+ return deadline << I915_SCHED_DEADLINE_SHIFT;
}
#endif /* _I915_SCHEDULER_H_ */
diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h
index bb5ab6ca7a80..032ccecbd951 100644
--- a/drivers/gpu/drm/i915/i915_scheduler_types.h
+++ b/drivers/gpu/drm/i915/i915_scheduler_types.h
@@ -69,6 +69,22 @@ struct i915_sched_node {
unsigned int flags;
#define I915_SCHED_HAS_EXTERNAL_CHAIN BIT(0)
intel_engine_mask_t semaphores;
+
+ /**
+ * @deadline: [virtual] deadline
+ *
+ * When the request is ready for execution, it is given a quota
+ * (the engine's timeslice) and a virtual deadline. The virtual
+ * deadline is derived from the current time:
+ * ktime_get() + (prio_ratio * timeslice)
+ *
+ * Requests are then executed in order of deadline completion.
+ * Requests with earlier deadlines than currently executing on
+ * the engine will preempt the active requests.
+ */
+ u64 deadline;
+#define I915_SCHED_DEADLINE_SHIFT 19 /* i.e. roughly 500us buckets */
+#define I915_DEADLINE_NEVER U64_MAX
};
struct i915_dependency {
@@ -81,6 +97,7 @@ struct i915_dependency {
#define I915_DEPENDENCY_ALLOC BIT(0)
#define I915_DEPENDENCY_EXTERNAL BIT(1)
#define I915_DEPENDENCY_WEAK BIT(2)
+ u64 ipi_deadline;
int ipi_priority;
};
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 92c628f18c60..db91e639918e 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -2124,6 +2124,7 @@ static int measure_preemption(struct intel_context *ce)
intel_ring_advance(rq, cs);
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
+ rq->sched.deadline = 0;
elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
i915_request_add(rq);
--
2.20.1
More information about the Intel-gfx-trybot
mailing list