[PATCH 25/27] drm/i915: Fair low-latency scheduling
Chris Wilson
chris at chris-wilson.co.uk
Sun Jun 7 18:58:22 UTC 2020
The first "scheduler" was a topographical sorting of requests into
priority order. The execution order was deterministic, the earliest
submitted, highest priority request would be executed first. Priority
inherited ensured that inversions were kept at bay, and allowed us to
dynamically boost priorities (e.g. for interactive pageflips).
The minimalistic timeslicing scheme was an attempt to introduce fairness
between long running requests, by evicting the active request at the end
of a timeslice and moving it to the back of its priority queue (while
ensuring that dependencies were kept in order). For short running
requests from many clients of equal priority, the scheme is still very
much FIFO submission ordering, and as unfair as before.
To impose fairness, we need an external metric that ensures that clients
are interpersed, we don't execute one long chain from client A before
executing any of client B. This could be imposed by the clients by using
a fences based on an external clock, that is they only submit work for a
"frame" at frame-interval, instead of submitting as much work as they
are able to. The standard SwapBuffers approach is akin to double
bufferring, where as one frame is being executed, the next is being
submitted, such that there is always a maximum of two frames per client
in the pipeline. Even this scheme exhibits unfairness under load as a
single client will execute two frames back to back before the next, and
with enough clients, deadlines will be missed.
The idea introduced by BFS/MuQSS is that fairness is introduced by
metering with an external clock. Every request, when it becomes ready to
execute is assigned a virtual deadline, and execution order is then
determined by earliest deadline. Priority is used as a hint, rather than
strict ordering, where high priority requests have earlier deadlines,
but not necessarily earlier than outstanding work. Thus work is executed
in order of 'readiness', with timeslicing to demote long running work.
The Achille's heel of this scheduler is its strong preference for
low-latency and favouring of new queues. Whereas it was easy to dominate
the old scheduler by flooding it with many requests over a short period
of time, the new scheduler can be dominated by a 'synchronous' client
that waits for each of its requests to complete before submitting the
next. As such a client has no history, it is always considered
ready-to-run and receives an earlier deadline than the long running
requests.
Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 12 +-
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 4 +-
drivers/gpu/drm/i915/gt/intel_engine_types.h | 24 --
drivers/gpu/drm/i915/gt/intel_lrc.c | 328 +++++++-----------
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 5 +-
drivers/gpu/drm/i915/gt/selftest_lrc.c | 43 ++-
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 6 +-
drivers/gpu/drm/i915/i915_priolist_types.h | 7 +-
drivers/gpu/drm/i915/i915_request.h | 4 +-
drivers/gpu/drm/i915/i915_scheduler.c | 317 ++++++++++++-----
drivers/gpu/drm/i915/i915_scheduler.h | 22 +-
drivers/gpu/drm/i915/i915_scheduler_types.h | 17 +
.../drm/i915/selftests/i915_mock_selftests.h | 1 +
drivers/gpu/drm/i915/selftests/i915_request.c | 1 +
.../gpu/drm/i915/selftests/i915_scheduler.c | 49 +++
15 files changed, 478 insertions(+), 362 deletions(-)
create mode 100644 drivers/gpu/drm/i915/selftests/i915_scheduler.c
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index d79307d790da..b99b3332467d 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -513,7 +513,6 @@ void intel_engine_init_execlists(struct intel_engine_cs *engine)
execlists->active =
memset(execlists->inflight, 0, sizeof(execlists->inflight));
- execlists->queue_priority_hint = INT_MIN;
execlists->queue = RB_ROOT_CACHED;
}
@@ -1188,14 +1187,15 @@ bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
}
}
-static int print_sched_attr(const struct i915_sched_attr *attr,
- char *buf, int x, int len)
+static int print_sched(const struct i915_sched_node *node,
+ char *buf, int x, int len)
{
- if (attr->priority == I915_PRIORITY_INVALID)
+ if (node->attr.priority == I915_PRIORITY_INVALID)
return x;
x += snprintf(buf + x, len - x,
- " prio=%d", attr->priority);
+ " prio=%d, dl=%llu",
+ node->attr.priority, node->deadline);
return x;
}
@@ -1208,7 +1208,7 @@ static void print_request(struct drm_printer *m,
char buf[80] = "";
int x = 0;
- x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
+ x = print_sched(&rq->sched, buf, x, sizeof(buf));
drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
prefix,
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index d0a1078ef632..ac9c777a6592 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -188,6 +188,7 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine)
i915_request_add_active_barriers(rq);
/* Install ourselves as a preemption barrier */
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */
/*
@@ -248,9 +249,6 @@ static int __engine_park(struct intel_wakeref *wf)
intel_engine_park_heartbeat(engine);
intel_engine_disarm_breadcrumbs(engine);
- /* Must be reset upon idling, or we may miss the busy wakeup. */
- GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
-
if (engine->park)
engine->park(engine);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 48e111f16dc5..a3c60038244c 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -231,30 +231,6 @@ struct intel_engine_execlists {
*/
unsigned int port_mask;
- /**
- * @switch_priority_hint: Second context priority.
- *
- * We submit multiple contexts to the HW simultaneously and would
- * like to occasionally switch between them to emulate timeslicing.
- * To know when timeslicing is suitable, we track the priority of
- * the context submitted second.
- */
- int switch_priority_hint;
-
- /**
- * @queue_priority_hint: Highest pending priority.
- *
- * When we add requests into the queue, or adjust the priority of
- * executing requests, we compute the maximum priority of those
- * pending requests. We can then use this value to determine if
- * we need to preempt the executing requests to service the queue.
- * However, since the we may have recorded the priority of an inflight
- * request we wanted to preempt but since completed, at the time of
- * dequeuing the priority hint may no longer may match the highest
- * available request priority.
- */
- int queue_priority_hint;
-
/**
* @queue: queue of requests, in priority lists
*/
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 635a444df2b4..8fc6d5bb7105 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -200,7 +200,7 @@ struct virtual_engine {
*/
struct ve_node {
struct rb_node rb;
- int prio;
+ u64 deadline;
} nodes[I915_NUM_ENGINES];
/*
@@ -411,12 +411,17 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb)
static inline int rq_prio(const struct i915_request *rq)
{
- return READ_ONCE(rq->sched.attr.priority);
+ return rq->sched.attr.priority;
}
-static int effective_prio(const struct i915_request *rq)
+static inline u64 rq_deadline(const struct i915_request *rq)
{
- int prio = rq_prio(rq);
+ return rq->sched.deadline;
+}
+
+static u64 effective_deadline(const struct i915_request *rq)
+{
+ u64 deadline = rq_deadline(rq);
/*
* If this request is special and must not be interrupted at any
@@ -427,27 +432,27 @@ static int effective_prio(const struct i915_request *rq)
* nopreempt for as long as desired).
*/
if (i915_request_has_nopreempt(rq))
- prio = I915_PRIORITY_UNPREEMPTABLE;
+ deadline = 0;
- return prio;
+ return deadline;
}
-static int queue_prio(const struct intel_engine_execlists *execlists)
+static u64 queue_deadline(const struct intel_engine_execlists *execlists)
{
struct rb_node *rb;
rb = rb_first_cached(&execlists->queue);
if (!rb)
- return INT_MIN;
+ return I915_DEADLINE_NEVER;
- return to_priolist(rb)->priority;
+ return to_priolist(rb)->deadline;
}
static inline bool need_preempt(const struct intel_engine_cs *engine,
const struct i915_request *rq,
struct virtual_engine *ve)
{
- int last_prio;
+ u64 last_deadline;
if (!intel_engine_has_semaphores(engine))
return false;
@@ -470,16 +475,14 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* priority level: the task that is running should remain running
* to preserve FIFO ordering of dependencies.
*/
- last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
- if (engine->execlists.queue_priority_hint <= last_prio)
- return false;
+ last_deadline = effective_deadline(rq);
/*
* Check against the first request in ELSP[1], it will, thanks to the
* power of PI, be the highest priority of that context.
*/
if (!list_is_last(&rq->sched.link, &engine->active.requests) &&
- rq_prio(list_next_entry(rq, sched.link)) > last_prio)
+ rq_deadline(list_next_entry(rq, sched.link)) < last_deadline)
return true;
if (ve) {
@@ -491,7 +494,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
rcu_read_lock();
next = READ_ONCE(ve->request);
if (next)
- preempt = rq_prio(next) > last_prio;
+ preempt = rq_deadline(next) < last_deadline;
rcu_read_unlock();
}
@@ -509,7 +512,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
* context, it's priority would not exceed ELSP[0] aka last_prio.
*/
- return queue_prio(&engine->execlists) > last_prio;
+ return queue_deadline(&engine->execlists) < last_deadline;
}
__maybe_unused static inline bool
@@ -526,7 +529,7 @@ assert_priority_queue(const struct i915_request *prev,
if (i915_request_is_active(prev))
return true;
- return rq_prio(prev) >= rq_prio(next);
+ return rq_deadline(prev) <= rq_deadline(next);
}
/*
@@ -1096,22 +1099,30 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
{
struct i915_request *rq, *rn, *active = NULL;
struct list_head *uninitialized_var(pl);
- int prio = I915_PRIORITY_INVALID;
+ u64 deadline = I915_DEADLINE_NEVER;
lockdep_assert_held(&engine->active.lock);
list_for_each_entry_safe_reverse(rq, rn,
&engine->active.requests,
sched.link) {
- if (i915_request_completed(rq))
+ if (i915_request_completed(rq)) {
+ list_del_init(&rq->sched.link);
continue; /* XXX */
+ }
__i915_request_unsubmit(rq);
- GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
- if (rq_prio(rq) != prio) {
- prio = rq_prio(rq);
- pl = i915_sched_lookup_priolist(engine, prio);
+ if (i915_request_started(rq)) {
+ u64 deadline =
+ i915_scheduler_next_virtual_deadline(rq_prio(rq));
+ rq->sched.deadline = min(rq_deadline(rq), deadline);
+ }
+
+ if (rq_deadline(rq) != deadline) {
+ deadline = rq_deadline(rq);
+ pl = i915_sched_lookup_priolist(engine, deadline);
+
}
GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
@@ -1546,14 +1557,14 @@ dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
if (!rq)
return "";
- snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
+ snprintf(buf, buflen, "%sccid:%x %llx:%lld%s dl %llu",
prefix,
rq->context->lrc.ccid,
rq->fence.context, rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
- rq_prio(rq));
+ rq_deadline(rq));
return buf;
}
@@ -1871,7 +1882,9 @@ static void virtual_xfer_breadcrumbs(struct virtual_engine *ve)
intel_engine_transfer_stale_breadcrumbs(ve->siblings[0], &ve->context);
}
-static void defer_request(struct i915_request *rq, struct list_head * const pl)
+static void defer_request(struct i915_request *rq,
+ struct list_head * const pl,
+ u64 deadline)
{
LIST_HEAD(list);
@@ -1886,6 +1899,7 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
struct i915_dependency *p;
GEM_BUG_ON(i915_request_is_active(rq));
+ rq->sched.deadline = deadline;
list_move_tail(&rq->sched.link, pl);
for_each_waiter(p, rq) {
@@ -1908,10 +1922,9 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
if (!i915_request_is_ready(w))
continue;
- if (rq_prio(w) < rq_prio(rq))
+ if (rq_deadline(w) > deadline)
continue;
- GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
list_move_tail(&w->sched.link, &list);
}
@@ -1922,46 +1935,21 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
static void defer_active(struct intel_engine_cs *engine)
{
struct i915_request *rq;
+ u64 deadline;
rq = __unwind_incomplete_requests(engine);
if (!rq)
return;
- defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq)));
-}
-
-static bool
-need_timeslice(const struct intel_engine_cs *engine,
- const struct i915_request *rq,
- struct virtual_engine *ve)
-{
- int hint;
-
- if (!intel_engine_has_timeslices(engine))
- return false;
-
- hint = engine->execlists.queue_priority_hint;
-
- if (ve) {
- const struct intel_engine_cs *inflight =
- intel_context_inflight(&ve->context);
-
- if (!inflight || inflight == engine) {
- struct i915_request *next;
-
- rcu_read_lock();
- next = READ_ONCE(ve->request);
- if (next)
- hint = max(hint, rq_prio(next));
- rcu_read_unlock();
- }
- }
-
- if (!list_is_last(&rq->sched.link, &engine->active.requests))
- hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
+ deadline = max(rq_deadline(rq),
+ i915_scheduler_next_virtual_deadline(rq_prio(rq)));
+ ENGINE_TRACE(engine, "defer %llx:%lld, dl:%llu -> %llu\n",
+ rq->fence.context, rq->fence.seqno,
+ rq_deadline(rq), deadline);
- GEM_BUG_ON(hint >= I915_PRIORITY_UNPREEMPTABLE);
- return hint >= effective_prio(rq);
+ defer_request(rq,
+ i915_sched_lookup_priolist(engine, deadline),
+ deadline);
}
static bool
@@ -1984,42 +1972,56 @@ timeslice_yield(const struct intel_engine_execlists *el,
}
static bool
-timeslice_expired(const struct intel_engine_execlists *el,
- const struct i915_request *rq)
+timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
{
- return timer_expired(&el->timer) || timeslice_yield(el, rq);
-}
+ const struct intel_engine_execlists *el = &engine->execlists;
-static int
-switch_prio(struct intel_engine_cs *engine, const struct i915_request *rq)
-{
- if (list_is_last(&rq->sched.link, &engine->active.requests))
- return engine->execlists.queue_priority_hint;
+ if (!intel_engine_has_timeslices(engine))
+ return false;
+
+ if (i915_request_has_nopreempt(rq) && i915_request_started(rq))
+ return false;
- return rq_prio(list_next_entry(rq, sched.link));
+ return timer_expired(&el->timer) || timeslice_yield(el, rq);
}
-static inline unsigned long
-timeslice(const struct intel_engine_cs *engine)
+static unsigned long timeslice(const struct intel_engine_cs *engine)
{
return READ_ONCE(engine->props.timeslice_duration_ms);
}
-static unsigned long active_timeslice(const struct intel_engine_cs *engine)
+static bool needs_timeslice(const struct intel_engine_cs *engine,
+ const struct i915_request *rq)
{
- const struct intel_engine_execlists *execlists = &engine->execlists;
- const struct i915_request *rq = *execlists->active;
-
+ /* If not currently active, or about to switch, wait for next event */
if (!rq || i915_request_completed(rq))
- return 0;
+ return false;
+
+ /* We do not need to start the timeslice until after the ACK */
+ if (READ_ONCE(engine->execlists.pending[0]))
+ return false;
+
+ /* If ELSP[1] is occupied, always check to see if worth slicing */
+ if (!list_is_last(&rq->sched.link, &engine->active.requests))
+ return true;
+
+ /* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */
+ if (rb_first_cached(&engine->execlists.queue))
+ return true;
- if (READ_ONCE(execlists->switch_priority_hint) < effective_prio(rq))
+ return rb_first_cached(&engine->execlists.virtual);
+}
+
+static unsigned long active_timeslice(const struct intel_engine_cs *engine)
+{
+ /* Disable the timer if there is nothing to switch to */
+ if (!needs_timeslice(engine, execlists_active(&engine->execlists)))
return 0;
return timeslice(engine);
}
-static void set_timeslice(struct intel_engine_cs *engine)
+static void start_timeslice(struct intel_engine_cs *engine)
{
unsigned long duration;
@@ -2032,29 +2034,6 @@ static void set_timeslice(struct intel_engine_cs *engine)
set_timer_ms(&engine->execlists.timer, duration);
}
-static void start_timeslice(struct intel_engine_cs *engine, int prio)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
- unsigned long duration;
-
- if (!intel_engine_has_timeslices(engine))
- return;
-
- WRITE_ONCE(execlists->switch_priority_hint, prio);
- if (prio == INT_MIN)
- return;
-
- if (timer_pending(&execlists->timer))
- return;
-
- duration = timeslice(engine);
- ENGINE_TRACE(engine,
- "start timeslicing, prio:%d, interval:%lu",
- prio, duration);
-
- set_timer_ms(&execlists->timer, duration);
-}
-
static void record_preemption(struct intel_engine_execlists *execlists)
{
(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
@@ -2146,11 +2125,10 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (need_preempt(engine, last, ve)) {
ENGINE_TRACE(engine,
- "preempting last=%llx:%lld, prio=%d, hint=%d\n",
+ "preempting last=%llx:%llu, dl=%llu\n",
last->fence.context,
last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint);
+ rq_deadline(last));
record_preemption(execlists);
/*
@@ -2170,14 +2148,13 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
__unwind_incomplete_requests(engine);
last = NULL;
- } else if (need_timeslice(engine, last, ve) &&
- timeslice_expired(execlists, last)) {
+ } else if (timeslice_expired(engine, last)) {
ENGINE_TRACE(engine,
- "expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
- last->fence.context,
- last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint,
+ "expired:%s last=%llx:%llu, deadline=%llu, now=%llu, yield?=%s\n",
+ yesno(timer_expired(&execlists->timer)),
+ last->fence.context, last->fence.seqno,
+ rq_deadline(last),
+ i915_sched_to_ticks(ktime_get()),
yesno(timeslice_yield(execlists, last)));
ring_set_paused(engine, 1);
@@ -2213,7 +2190,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* Even if ELSP[1] is occupied and not worthy
* of timeslices, our queue might be.
*/
- start_timeslice(engine, queue_prio(execlists));
return;
}
}
@@ -2232,7 +2208,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
GEM_BUG_ON(rq->engine != &ve->base);
GEM_BUG_ON(rq->context != &ve->context);
- if (unlikely(rq_prio(rq) < queue_prio(execlists))) {
+ if (unlikely(rq_deadline(rq) > queue_deadline(execlists))) {
spin_unlock(&ve->base.active.lock);
break;
}
@@ -2241,7 +2217,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (last && !can_merge_rq(last, rq)) {
spin_unlock(&ve->base.active.lock);
- start_timeslice(engine, rq_prio(rq));
return; /* leave this for another sibling */
}
@@ -2253,10 +2228,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
i915_request_started(rq) ? "*" :
"",
yesno(engine != ve->siblings[0]));
-
WRITE_ONCE(ve->request, NULL);
- WRITE_ONCE(ve->base.execlists.queue_priority_hint,
- INT_MIN);
rb = &ve->nodes[engine->id].rb;
rb_erase_cached(rb, &execlists->virtual);
@@ -2399,28 +2371,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
}
done:
- /*
- * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
- *
- * We choose the priority hint such that if we add a request of greater
- * priority than this, we kick the submission tasklet to decide on
- * the right order of submitting the requests to hardware. We must
- * also be prepared to reorder requests as they are in-flight on the
- * HW. We derive the priority hint then as the first "hole" in
- * the HW submission ports and if there are no available slots,
- * the priority of the lowest executing request, i.e. last.
- *
- * When we do receive a higher priority request ready to run from the
- * user, see queue_request(), the priority hint is bumped to that
- * request triggering preemption on the next dequeue (or subsequent
- * interrupt for secondary ports).
- */
- execlists->queue_priority_hint = queue_prio(execlists);
-
if (submit) {
*port = execlists_schedule_in(last, port - execlists->pending);
- execlists->switch_priority_hint =
- switch_prio(engine, *execlists->pending);
/*
* Skip if we ended up with exactly the same set of requests,
@@ -2440,7 +2392,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
set_preempt_timeout(engine, *active);
execlists_submit_ports(engine);
} else {
- start_timeslice(engine, execlists->queue_priority_hint);
skip_submit:
ring_set_paused(engine, 0);
}
@@ -2683,7 +2634,6 @@ static void process_csb(struct intel_engine_cs *engine)
} while (head != tail);
execlists->csb_head = head;
- set_timeslice(engine);
/*
* Gen11 has proven to fail wrt global observation point between
@@ -2832,9 +2782,10 @@ static bool hold_request(const struct i915_request *rq)
return result;
}
-static void __execlists_unhold(struct i915_request *rq)
+static bool __execlists_unhold(struct i915_request *rq)
{
LIST_HEAD(list);
+ bool submit = false;
do {
struct i915_dependency *p;
@@ -2845,10 +2796,7 @@ static void __execlists_unhold(struct i915_request *rq)
GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
i915_request_clear_hold(rq);
- list_move_tail(&rq->sched.link,
- i915_sched_lookup_priolist(rq->engine,
- rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ submit |= intel_engine_queue_request(rq->engine, rq);
/* Also release any children on this engine that are ready */
for_each_waiter(p, rq) {
@@ -2877,6 +2825,8 @@ static void __execlists_unhold(struct i915_request *rq)
rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
} while (rq);
+
+ return submit;
}
static void execlists_unhold(struct intel_engine_cs *engine,
@@ -2888,12 +2838,8 @@ static void execlists_unhold(struct intel_engine_cs *engine,
* Move this request back to the priority queue, and all of its
* children and grandchildren that were suspended along with it.
*/
- __execlists_unhold(rq);
-
- if (rq_prio(rq) > engine->execlists.queue_priority_hint) {
- engine->execlists.queue_priority_hint = rq_prio(rq);
+ if (__execlists_unhold(rq))
tasklet_hi_schedule(&engine->execlists.tasklet);
- }
spin_unlock_irq(&engine->active.lock);
}
@@ -3135,6 +3081,8 @@ static void execlists_submission_tasklet(unsigned long data)
if (unlikely(timeout && preempt_timeout(engine)))
execlists_reset(engine, "preemption time out");
}
+
+ start_timeslice(engine);
}
static void __execlists_kick(struct intel_engine_execlists *execlists)
@@ -3156,15 +3104,6 @@ static void execlists_preempt(struct timer_list *timer)
execlists_kick(timer, preempt);
}
-static void queue_request(struct intel_engine_cs *engine,
- struct i915_request *rq)
-{
- GEM_BUG_ON(!list_empty(&rq->sched.link));
- list_add_tail(&rq->sched.link,
- i915_sched_lookup_priolist(engine, rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
-}
-
static void __submit_queue_imm(struct intel_engine_cs *engine)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -3175,18 +3114,6 @@ static void __submit_queue_imm(struct intel_engine_cs *engine)
__execlists_submission_tasklet(engine);
}
-static void submit_queue(struct intel_engine_cs *engine,
- const struct i915_request *rq)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
-
- if (rq_prio(rq) <= execlists->queue_priority_hint)
- return;
-
- execlists->queue_priority_hint = rq_prio(rq);
- __submit_queue_imm(engine);
-}
-
static bool ancestor_on_hold(const struct intel_engine_cs *engine,
const struct i915_request *rq)
{
@@ -3221,12 +3148,9 @@ static void execlists_submit_request(struct i915_request *request)
list_add_tail(&request->sched.link, &engine->active.hold);
i915_request_set_hold(request);
} else {
- queue_request(engine, request);
-
- GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
- GEM_BUG_ON(list_empty(&request->sched.link));
-
- submit_queue(engine, request);
+ if (intel_engine_queue_request(engine, request))
+ __submit_queue_imm(engine);
+ start_timeslice(engine);
}
spin_unlock_irqrestore(&engine->active.lock, flags);
@@ -4281,10 +4205,6 @@ static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled)
static void nop_submission_tasklet(unsigned long data)
{
- struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
-
- /* The driver is wedged; don't process any more events. */
- WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN);
}
static void execlists_reset_cancel(struct intel_engine_cs *engine)
@@ -4330,6 +4250,7 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine)
rb_erase_cached(&p->node, &execlists->queue);
i915_priolist_free(p);
}
+ GEM_BUG_ON(!RB_EMPTY_ROOT(&execlists->queue.rb_root));
/* On-hold requests will be flushed to timeline upon their release */
list_for_each_entry(rq, &engine->active.hold, sched.link)
@@ -4351,17 +4272,12 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine)
rq->engine = engine;
__i915_request_submit(rq);
i915_request_put(rq);
-
- ve->base.execlists.queue_priority_hint = INT_MIN;
}
spin_unlock(&ve->base.active.lock);
}
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
- execlists->queue = RB_ROOT_CACHED;
-
GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
execlists->tasklet.func = nop_submission_tasklet;
@@ -5457,7 +5373,8 @@ static const struct intel_context_ops virtual_context_ops = {
.destroy = virtual_context_destroy,
};
-static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
+static intel_engine_mask_t
+virtual_submission_mask(struct virtual_engine *ve, u64 *deadline)
{
struct i915_request *rq;
intel_engine_mask_t mask;
@@ -5474,9 +5391,11 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
mask = ve->siblings[0]->mask;
}
- ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
+ *deadline = rq_deadline(rq);
+
+ ENGINE_TRACE(&ve->base, "rq=%llx:%llu, mask=%x, dl=%llu\n",
rq->fence.context, rq->fence.seqno,
- mask, ve->base.execlists.queue_priority_hint);
+ mask, *deadline);
return mask;
}
@@ -5484,12 +5403,12 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
static void virtual_submission_tasklet(unsigned long data)
{
struct virtual_engine * const ve = (struct virtual_engine *)data;
- const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint);
intel_engine_mask_t mask;
+ u64 deadline;
unsigned int n;
rcu_read_lock();
- mask = virtual_submission_mask(ve);
+ mask = virtual_submission_mask(ve, &deadline);
rcu_read_unlock();
if (unlikely(!mask))
return;
@@ -5522,7 +5441,8 @@ static void virtual_submission_tasklet(unsigned long data)
*/
first = rb_first_cached(&sibling->execlists.virtual) ==
&node->rb;
- if (prio == node->prio || (prio > node->prio && first))
+ if (deadline == node->deadline ||
+ (deadline < node->deadline && first))
goto submit_engine;
rb_erase_cached(&node->rb, &sibling->execlists.virtual);
@@ -5536,7 +5456,7 @@ static void virtual_submission_tasklet(unsigned long data)
rb = *parent;
other = rb_entry(rb, typeof(*other), rb);
- if (prio > other->prio) {
+ if (deadline < other->deadline) {
parent = &rb->rb_left;
} else {
parent = &rb->rb_right;
@@ -5551,8 +5471,8 @@ static void virtual_submission_tasklet(unsigned long data)
submit_engine:
GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
- node->prio = prio;
- if (first && prio > sibling->execlists.queue_priority_hint)
+ node->deadline = deadline;
+ if (first)
tasklet_hi_schedule(&sibling->execlists.tasklet);
unlock_engine:
@@ -5586,11 +5506,11 @@ static void virtual_submit_request(struct i915_request *rq)
if (i915_request_completed(rq)) {
__i915_request_submit(rq);
-
- ve->base.execlists.queue_priority_hint = INT_MIN;
ve->request = NULL;
} else {
- ve->base.execlists.queue_priority_hint = rq_prio(rq);
+ rq->sched.deadline =
+ min(rq->sched.deadline,
+ i915_scheduler_next_virtual_deadline(rq_prio(rq)));
ve->request = i915_request_get(rq);
GEM_BUG_ON(!list_empty(virtual_queue(ve)));
@@ -5694,7 +5614,6 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings,
ve->base.bond_execute = virtual_bond_execute;
INIT_LIST_HEAD(virtual_queue(ve));
- ve->base.execlists.queue_priority_hint = INT_MIN;
tasklet_init(&ve->base.execlists.tasklet,
virtual_submission_tasklet,
(unsigned long)ve);
@@ -5881,13 +5800,6 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine,
show_request(m, last, "\t\tE ");
}
- if (execlists->switch_priority_hint != INT_MIN)
- drm_printf(m, "\t\tSwitch priority hint: %d\n",
- READ_ONCE(execlists->switch_priority_hint));
- if (execlists->queue_priority_hint != INT_MIN)
- drm_printf(m, "\t\tQueue priority hint: %d\n",
- READ_ONCE(execlists->queue_priority_hint));
-
last = NULL;
count = 0;
for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index afa4f88035ac..01fca8acd4c4 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -879,7 +879,10 @@ static int __igt_reset_engines(struct intel_gt *gt,
break;
}
- if (i915_request_wait(rq, 0, HZ / 5) < 0) {
+ /* With deadlines, no strict priority */
+ i915_request_set_deadline(rq, 0);
+
+ if (i915_request_wait(rq, 0, HZ / 2) < 0) {
struct drm_printer p =
drm_info_printer(gt->i915->drm.dev);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 052dcc59fcc5..b18276cf30ed 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -85,6 +85,9 @@ static int wait_for_submit(struct intel_engine_cs *engine,
struct i915_request *rq,
unsigned long timeout)
{
+ /* Ignore our own attempts to suppress excess tasklets */
+ tasklet_hi_schedule(&engine->execlists.tasklet);
+
timeout += jiffies;
do {
bool done = time_after(jiffies, timeout);
@@ -754,7 +757,7 @@ semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
static int
release_queue(struct intel_engine_cs *engine,
struct i915_vma *vma,
- int idx, int prio)
+ int idx, u64 deadline)
{
struct i915_request *rq;
u32 *cs;
@@ -779,10 +782,7 @@ release_queue(struct intel_engine_cs *engine,
i915_request_get(rq);
i915_request_add(rq);
- local_bh_disable();
- i915_request_set_priority(rq, prio);
- local_bh_enable(); /* kick tasklet */
-
+ i915_request_set_deadline(rq, deadline);
i915_request_put(rq);
return 0;
@@ -796,6 +796,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
struct intel_engine_cs *engine;
struct i915_request *head;
enum intel_engine_id id;
+ long timeout;
int err, i, n = 0;
head = semaphore_queue(outer, vma, n++);
@@ -816,12 +817,16 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
}
}
- err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
+ err = release_queue(outer, vma, n, 0);
if (err)
goto out;
- if (i915_request_wait(head, 0,
- 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
+ /* Expected number of pessimal slices required */
+ timeout = RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3);
+ timeout *= 4; /* safety factor, including bucketing */
+ timeout += HZ / 2; /* and include the request completion */
+
+ if (i915_request_wait(head, 0, timeout) < 0) {
pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
count, n);
GEM_TRACE_DUMP();
@@ -926,6 +931,8 @@ create_rewinder(struct intel_context *ce,
err = i915_request_await_dma_fence(rq, &wait->fence);
if (err)
goto err;
+
+ i915_request_set_deadline(rq, rq_deadline(wait));
}
cs = intel_ring_begin(rq, 14);
@@ -1200,7 +1207,7 @@ static int live_timeslice_queue(void *arg)
err = PTR_ERR(rq);
goto err_heartbeat;
}
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
err = wait_for_submit(engine, rq, HZ / 2);
if (err) {
pr_err("%s: Timed out trying to submit semaphores\n",
@@ -1223,10 +1230,9 @@ static int live_timeslice_queue(void *arg)
}
GEM_BUG_ON(i915_request_completed(rq));
- GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
/* Queue: semaphore signal, matching priority as semaphore */
- err = release_queue(engine, vma, 1, effective_prio(rq));
+ err = release_queue(engine, vma, 1, effective_deadline(rq));
if (err)
goto err_rq;
@@ -1326,7 +1332,7 @@ static int live_timeslice_nopreempt(void *arg)
ce = intel_context_create(engine);
if (IS_ERR(ce)) {
- err = PTR_ERR(rq);
+ err = PTR_ERR(ce);
goto out_spin;
}
@@ -1337,6 +1343,7 @@ static int live_timeslice_nopreempt(void *arg)
goto out_spin;
}
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
i915_request_get(rq);
i915_request_add(rq);
@@ -1709,6 +1716,7 @@ static int live_late_preempt(void *arg)
/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
ctx_lo->sched.priority = 1;
+ ctx_hi->sched.priority = I915_PRIORITY_MIN;
for_each_engine(engine, gt, id) {
struct igt_live_test t;
@@ -2648,6 +2656,9 @@ static int live_preempt_gang(void *arg)
struct i915_request *n =
list_next_entry(rq, client_link);
+ /* With deadlines, no strict priority ordering */
+ i915_request_set_deadline(rq, 0);
+
if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
struct drm_printer p =
drm_info_printer(engine->i915->drm.dev);
@@ -2869,7 +2880,7 @@ static int preempt_user(struct intel_engine_cs *engine,
i915_request_get(rq);
i915_request_add(rq);
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
if (i915_request_wait(rq, 0, HZ / 2) < 0)
err = -ETIME;
@@ -4402,6 +4413,7 @@ static int emit_semaphore_signal(struct intel_context *ce, void *slot)
intel_ring_advance(rq, cs);
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
i915_request_add(rq);
return 0;
@@ -4911,6 +4923,10 @@ static int __live_lrc_gpr(struct intel_engine_cs *engine,
err = emit_semaphore_signal(engine->kernel_context, slot);
if (err)
goto err_rq;
+
+ err = wait_for_submit(engine, rq, HZ / 2);
+ if (err)
+ goto err_rq;
} else {
slot[0] = 1;
wmb();
@@ -5468,6 +5484,7 @@ static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
intel_ring_advance(rq, cs);
+ rq->sched.deadline = 0;
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
err_rq:
i915_request_add(rq);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 0c42e8b0c211..6da465c7c4f5 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -333,8 +333,6 @@ static void __guc_dequeue(struct intel_engine_cs *engine)
i915_priolist_free(p);
}
done:
- execlists->queue_priority_hint =
- rb ? to_priolist(rb)->priority : INT_MIN;
if (submit) {
*port = schedule_in(last, port - execlists->inflight);
*++port = NULL;
@@ -473,12 +471,10 @@ static void guc_reset_cancel(struct intel_engine_cs *engine)
rb_erase_cached(&p->node, &execlists->queue);
i915_priolist_free(p);
}
+ GEM_BUG_ON(!RB_EMPTY_ROOT(&execlists->queue.rb_root));
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
- execlists->queue = RB_ROOT_CACHED;
-
spin_unlock_irqrestore(&engine->active.lock, flags);
}
diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h
index bc2fa84f98a8..43a0ac45295f 100644
--- a/drivers/gpu/drm/i915/i915_priolist_types.h
+++ b/drivers/gpu/drm/i915/i915_priolist_types.h
@@ -22,6 +22,8 @@ enum {
/* Interactive workload, scheduled for immediate pageflipping */
I915_PRIORITY_DISPLAY,
+
+ __I915_PRIORITY_KERNEL__
};
/* Smallest priority value that cannot be bumped. */
@@ -35,13 +37,12 @@ enum {
* i.e. nothing can have higher priority and force us to usurp the
* active request.
*/
-#define I915_PRIORITY_UNPREEMPTABLE INT_MAX
-#define I915_PRIORITY_BARRIER (I915_PRIORITY_UNPREEMPTABLE - 1)
+#define I915_PRIORITY_BARRIER INT_MAX
struct i915_priolist {
struct list_head requests;
struct rb_node node;
- int priority;
+ u64 deadline;
};
#endif /* _I915_PRIOLIST_TYPES_H_ */
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 118ab6650d1f..23594e712292 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -561,7 +561,7 @@ static inline void i915_request_clear_hold(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_timeline(struct i915_request *rq)
+i915_request_timeline(const struct i915_request *rq)
{
/* Valid only while the request is being constructed (or retired). */
return rcu_dereference_protected(rq->timeline,
@@ -576,7 +576,7 @@ i915_request_gem_context(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_active_timeline(struct i915_request *rq)
+i915_request_active_timeline(const struct i915_request *rq)
{
/*
* When in use during submission, we are protected by a guarantee that
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index 4c189b81cc62..ca2afad6dd2b 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -20,6 +20,11 @@ static struct i915_global_scheduler {
static DEFINE_SPINLOCK(ipi_lock);
static LIST_HEAD(ipi_list);
+static inline u64 rq_deadline(const struct i915_request *rq)
+{
+ return READ_ONCE(rq->sched.deadline);
+}
+
static inline int rq_prio(const struct i915_request *rq)
{
return READ_ONCE(rq->sched.attr.priority);
@@ -32,6 +37,7 @@ static void ipi_schedule(struct irq_work *wrk)
struct i915_dependency *p;
struct i915_request *rq;
unsigned long flags;
+ u64 deadline;
int prio;
spin_lock_irqsave(&ipi_lock, flags);
@@ -40,7 +46,10 @@ static void ipi_schedule(struct irq_work *wrk)
rq = container_of(p->signaler, typeof(*rq), sched);
list_del_init(&p->ipi_link);
+ deadline = p->ipi_deadline;
prio = p->ipi_priority;
+
+ p->ipi_deadline = I915_DEADLINE_NEVER;
p->ipi_priority = I915_PRIORITY_INVALID;
}
spin_unlock_irqrestore(&ipi_lock, flags);
@@ -52,6 +61,8 @@ static void ipi_schedule(struct irq_work *wrk)
if (prio > rq_prio(rq))
i915_request_set_priority(rq, prio);
+ if (deadline < rq_deadline(rq))
+ i915_request_set_deadline(rq, deadline);
} while (1);
rcu_read_unlock();
}
@@ -79,28 +90,8 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb)
return rb_entry(rb, struct i915_priolist, node);
}
-static void assert_priolists(struct intel_engine_execlists * const execlists)
-{
- struct rb_node *rb;
- long last_prio;
-
- if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
- return;
-
- GEM_BUG_ON(rb_first_cached(&execlists->queue) !=
- rb_first(&execlists->queue.rb_root));
-
- last_prio = INT_MAX;
- for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
- const struct i915_priolist *p = to_priolist(rb);
-
- GEM_BUG_ON(p->priority > last_prio);
- last_prio = p->priority;
- }
-}
-
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
struct i915_priolist *p;
@@ -108,10 +99,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
bool first = true;
lockdep_assert_held(&engine->active.lock);
- assert_priolists(execlists);
if (unlikely(execlists->no_priolist))
- prio = I915_PRIORITY_NORMAL;
+ deadline = 0;
find_priolist:
/* most positive priority is scheduled first, equal priorities fifo */
@@ -120,9 +110,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
while (*parent) {
rb = *parent;
p = to_priolist(rb);
- if (prio > p->priority) {
+ if (deadline < p->deadline) {
parent = &rb->rb_left;
- } else if (prio < p->priority) {
+ } else if (deadline > p->deadline) {
parent = &rb->rb_right;
first = false;
} else {
@@ -130,13 +120,13 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- if (prio == I915_PRIORITY_NORMAL) {
+ if (!deadline) {
p = &execlists->default_priolist;
} else {
p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC);
/* Convert an allocation failure to a priority bump */
if (unlikely(!p)) {
- prio = I915_PRIORITY_NORMAL; /* recurses just once */
+ deadline = 0; /* recurses just once */
/* To maintain ordering with all rendering, after an
* allocation failure we have to disable all scheduling.
@@ -151,7 +141,7 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- p->priority = prio;
+ p->deadline = deadline;
INIT_LIST_HEAD(&p->requests);
rb_link_node(&p->node, rb, parent);
@@ -160,70 +150,216 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
return &p->requests;
}
-void __i915_priolist_free(struct i915_priolist *p)
+void i915_priolist_free(struct i915_priolist *p)
+{
+ if (p->deadline)
+ kmem_cache_free(global.slab_priorities, p);
+}
+
+static bool kick_submission(const struct intel_engine_cs *engine, u64 deadline)
{
- kmem_cache_free(global.slab_priorities, p);
+ const struct i915_request *inflight;
+ bool kick = true;
+
+ rcu_read_lock();
+ inflight = execlists_active(&engine->execlists);
+ if (inflight)
+ kick = deadline < rq_deadline(inflight);
+ rcu_read_unlock();
+
+ return kick;
}
-static inline bool need_preempt(int prio, int active)
+static bool __i915_request_set_deadline(struct i915_request *rq, u64 deadline)
{
+ struct intel_engine_cs *engine = rq->engine;
+ struct i915_request *rn;
+ struct list_head *plist;
+ LIST_HEAD(dfs);
+
+ lockdep_assert_held(&engine->active.lock);
+ list_add(&rq->sched.dfs, &dfs);
+
+ list_for_each_entry(rq, &dfs, sched.dfs) {
+ struct i915_dependency *p;
+
+ GEM_BUG_ON(rq->engine != engine);
+
+ for_each_signaler(p, rq) {
+ struct i915_request *s =
+ container_of(p->signaler, typeof(*s), sched);
+
+ GEM_BUG_ON(s == rq);
+
+ if (rq_deadline(s) <= deadline)
+ continue;
+
+ if (i915_request_completed(s))
+ continue;
+
+ if (s->engine != rq->engine) {
+ spin_lock(&ipi_lock);
+ if (deadline < p->ipi_deadline) {
+ p->ipi_deadline = deadline;
+ list_move(&p->ipi_link, &ipi_list);
+ irq_work_queue(&ipi_work);
+ }
+ spin_unlock(&ipi_lock);
+ continue;
+ }
+
+ list_move_tail(&s->sched.dfs, &dfs);
+ }
+ }
+
+ plist = i915_sched_lookup_priolist(engine, deadline);
+
+ /* Fifo and depth-first replacement ensure our deps execute first */
+ list_for_each_entry_safe_reverse(rq, rn, &dfs, sched.dfs) {
+ GEM_BUG_ON(rq->engine != engine);
+ GEM_BUG_ON(deadline > rq_deadline(rq));
+
+ INIT_LIST_HEAD(&rq->sched.dfs);
+ WRITE_ONCE(rq->sched.deadline, deadline);
+ RQ_TRACE(rq, "set-deadline:%llu\n", deadline);
+
+ /*
+ * Once the request is ready, it will be placed into the
+ * priority lists and then onto the HW runlist. Before the
+ * request is ready, it does not contribute to our preemption
+ * decisions and we can safely ignore it, as it will, and
+ * any preemption required, be dealt with upon submission.
+ * See engine->submit_request()
+ */
+
+ if (i915_request_in_priority_queue(rq))
+ list_move_tail(&rq->sched.link, plist);
+ }
+
+ return kick_submission(engine, deadline);
+}
+
+void i915_request_set_deadline(struct i915_request *rq, u64 deadline)
+{
+ struct intel_engine_cs *engine = READ_ONCE(rq->engine);
+ unsigned long flags;
+
+ if (!intel_engine_has_scheduler(engine))
+ return;
+
/*
- * Allow preemption of low -> normal -> high, but we do
- * not allow low priority tasks to preempt other low priority
- * tasks under the impression that latency for low priority
- * tasks does not matter (as much as background throughput),
- * so kiss.
+ * Virtual engines complicate acquiring the engine timeline lock,
+ * as their rq->engine pointer is not stable until under that
+ * engine lock. The simple ploy we use is to take the lock then
+ * check that the rq still belongs to the newly locked engine.
*/
- return prio >= max(I915_PRIORITY_NORMAL, active);
+ spin_lock_irqsave(&engine->active.lock, flags);
+ while (engine != READ_ONCE(rq->engine)) {
+ spin_unlock(&engine->active.lock);
+ engine = READ_ONCE(rq->engine);
+ spin_lock(&engine->active.lock);
+ }
+
+ if (i915_request_completed(rq))
+ goto unlock;
+
+ if (deadline >= rq_deadline(rq))
+ goto unlock;
+
+ if (__i915_request_set_deadline(rq, deadline))
+ tasklet_hi_schedule(&engine->execlists.tasklet);
+
+unlock:
+ spin_unlock_irqrestore(&engine->active.lock, flags);
}
-static void kick_submission(struct intel_engine_cs *engine,
- const struct i915_request *rq,
- int prio)
+static u64 prio_slice(int prio)
{
- const struct i915_request *inflight;
+ u64 slice;
+ int sf;
/*
- * We only need to kick the tasklet once for the high priority
- * new context we add into the queue.
+ * With a 1ms scheduling quantum:
+ *
+ * MAX USER: ~32us deadline
+ * 0: ~16ms deadline
+ * MIN_USER: 1000ms deadline
*/
- if (prio <= engine->execlists.queue_priority_hint)
- return;
- rcu_read_lock();
+ if (prio >= __I915_PRIORITY_KERNEL__)
+ return INT_MAX - prio;
- /* Nothing currently active? We're overdue for a submission! */
- inflight = execlists_active(&engine->execlists);
- if (!inflight)
- goto unlock;
+ slice = __I915_PRIORITY_KERNEL__ - prio;
+ if (prio >= 0)
+ sf = 20 - 6;
+ else
+ sf = 20 - 1;
+
+ return slice << sf;
+}
+
+u64 i915_scheduler_virtual_deadline(u64 kt, int priority)
+{
+ return i915_sched_to_ticks(kt + prio_slice(priority));
+}
+
+u64 i915_scheduler_next_virtual_deadline(int priority)
+{
+ return i915_scheduler_virtual_deadline(ktime_get(), priority);
+}
+
+static u64 signal_deadline(const struct i915_request *rq)
+{
+ u64 last = ktime_to_ns(ktime_get());
+ const struct i915_dependency *p;
/*
- * If we are already the currently executing context, don't
- * bother evaluating if we should preempt ourselves.
+ * Find the earliest point at which we will become 'ready',
+ * which we infer from the deadline of all active signalers.
+ * We will position ourselves at the end of that chain of work.
*/
- if (inflight->context == rq->context)
- goto unlock;
- ENGINE_TRACE(engine,
- "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
- prio,
- rq->fence.context, rq->fence.seqno,
- inflight->fence.context, inflight->fence.seqno,
- inflight->sched.attr.priority);
+ rcu_read_lock();
+ for_each_signaler(p, rq) {
+ const struct i915_request *s =
+ container_of(p->signaler, typeof(*s), sched);
- engine->execlists.queue_priority_hint = prio;
- if (need_preempt(prio, rq_prio(inflight)))
- tasklet_hi_schedule(&engine->execlists.tasklet);
+ if (i915_request_completed(s))
+ continue;
-unlock:
+ if (rq_prio(s) < rq_prio(rq))
+ continue;
+
+ last = max(last, i915_sched_to_ns(rq_deadline(s)));
+ }
rcu_read_unlock();
+
+ return last;
}
-static void __i915_request_set_priority(struct i915_request *rq, int prio)
+static u64 earliest_deadline(const struct i915_request *rq)
+{
+ return i915_scheduler_virtual_deadline(signal_deadline(rq),
+ rq_prio(rq));
+}
+
+static bool set_earliest_deadline(struct i915_request *rq, u64 old)
+{
+ u64 dl;
+
+ /* Recompute our deadlines and promote after a priority change */
+ dl = min(earliest_deadline(rq), rq_deadline(rq));
+ if (dl >= old)
+ return false;
+
+ return __i915_request_set_deadline(rq, dl);
+}
+
+static bool __i915_request_set_priority(struct i915_request *rq, int prio)
{
struct intel_engine_cs *engine = rq->engine;
struct i915_request *rn;
- struct list_head *plist;
+ bool kick = false;
LIST_HEAD(dfs);
lockdep_assert_held(&engine->active.lock);
@@ -280,32 +416,20 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
}
}
- plist = i915_sched_lookup_priolist(engine, prio);
-
- /* Fifo and depth-first replacement ensure our deps execute first */
list_for_each_entry_safe_reverse(rq, rn, &dfs, sched.dfs) {
GEM_BUG_ON(rq->engine != engine);
+ GEM_BUG_ON(prio < rq_prio(rq));
INIT_LIST_HEAD(&rq->sched.dfs);
WRITE_ONCE(rq->sched.attr.priority, prio);
+ RQ_TRACE(rq, "set-priority:%d\n", prio);
- /*
- * Once the request is ready, it will be placed into the
- * priority lists and then onto the HW runlist. Before the
- * request is ready, it does not contribute to our preemption
- * decisions and we can safely ignore it, as it will, and
- * any preemption required, be dealt with upon submission.
- * See engine->submit_request()
- */
- if (!i915_request_is_ready(rq))
- continue;
-
- if (i915_request_in_priority_queue(rq))
- list_move_tail(&rq->sched.link, plist);
-
- /* Defer (tasklet) submission until after all updates. */
- kick_submission(engine, rq, prio);
+ if (i915_request_is_ready(rq) &&
+ set_earliest_deadline(rq, rq_deadline(rq)))
+ kick = true;
}
+
+ return kick;
}
void i915_request_set_priority(struct i915_request *rq, int prio)
@@ -316,12 +440,6 @@ void i915_request_set_priority(struct i915_request *rq, int prio)
if (!intel_engine_has_scheduler(engine))
return;
- /*
- * Virtual engines complicate acquiring the engine timeline lock,
- * as their rq->engine pointer is not stable until under that
- * engine lock. The simple ploy we use is to take the lock then
- * check that the rq still belongs to the newly locked engine.
- */
spin_lock_irqsave(&engine->active.lock, flags);
while (engine != READ_ONCE(rq->engine)) {
spin_unlock(&engine->active.lock);
@@ -335,12 +453,21 @@ void i915_request_set_priority(struct i915_request *rq, int prio)
if (prio <= rq_prio(rq))
goto unlock;
- __i915_request_set_priority(rq, prio);
+ if (__i915_request_set_priority(rq, prio))
+ tasklet_hi_schedule(&engine->execlists.tasklet);
unlock:
spin_unlock_irqrestore(&engine->active.lock, flags);
}
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq)
+{
+ lockdep_assert_held(&engine->active.lock);
+ set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ return set_earliest_deadline(rq, I915_DEADLINE_NEVER);
+}
+
void i915_sched_node_init(struct i915_sched_node *node)
{
spin_lock_init(&node->lock);
@@ -356,6 +483,7 @@ void i915_sched_node_init(struct i915_sched_node *node)
void i915_sched_node_reinit(struct i915_sched_node *node)
{
node->attr.priority = I915_PRIORITY_INVALID;
+ node->deadline = I915_DEADLINE_NEVER;
node->semaphores = 0;
node->flags = 0;
@@ -388,6 +516,7 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
if (!node_signaled(signal)) {
INIT_LIST_HEAD(&dep->ipi_link);
+ dep->ipi_deadline = I915_DEADLINE_NEVER;
dep->ipi_priority = I915_PRIORITY_INVALID;
dep->signaler = signal;
dep->waiter = node;
@@ -519,6 +648,10 @@ void i915_sched_node_retire(struct i915_sched_node *node)
spin_unlock_irq(&node->lock);
}
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/i915_scheduler.c"
+#endif
+
static void i915_global_scheduler_shrink(void)
{
kmem_cache_shrink(global.slab_dependencies);
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index b26a13ef6feb..62265108230f 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -37,15 +37,27 @@ int i915_sched_node_add_dependency(struct i915_sched_node *node,
void i915_sched_node_retire(struct i915_sched_node *node);
void i915_request_set_priority(struct i915_request *request, int prio);
+void i915_request_set_deadline(struct i915_request *request, u64 deadline);
+
+u64 i915_scheduler_virtual_deadline(u64 kt, int priority);
+u64 i915_scheduler_next_virtual_deadline(int priority);
+
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq);
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio);
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline);
+
+void i915_priolist_free(struct i915_priolist *p);
+
+static inline u64 i915_sched_to_ticks(ktime_t kt)
+{
+ return ktime_to_ns(kt) >> I915_SCHED_DEADLINE_SHIFT;
+}
-void __i915_priolist_free(struct i915_priolist *p);
-static inline void i915_priolist_free(struct i915_priolist *p)
+static inline u64 i915_sched_to_ns(u64 deadline)
{
- if (p->priority != I915_PRIORITY_NORMAL)
- __i915_priolist_free(p);
+ return deadline << I915_SCHED_DEADLINE_SHIFT;
}
#endif /* _I915_SCHEDULER_H_ */
diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h
index bb5ab6ca7a80..032ccecbd951 100644
--- a/drivers/gpu/drm/i915/i915_scheduler_types.h
+++ b/drivers/gpu/drm/i915/i915_scheduler_types.h
@@ -69,6 +69,22 @@ struct i915_sched_node {
unsigned int flags;
#define I915_SCHED_HAS_EXTERNAL_CHAIN BIT(0)
intel_engine_mask_t semaphores;
+
+ /**
+ * @deadline: [virtual] deadline
+ *
+ * When the request is ready for execution, it is given a quota
+ * (the engine's timeslice) and a virtual deadline. The virtual
+ * deadline is derived from the current time:
+ * ktime_get() + (prio_ratio * timeslice)
+ *
+ * Requests are then executed in order of deadline completion.
+ * Requests with earlier deadlines than currently executing on
+ * the engine will preempt the active requests.
+ */
+ u64 deadline;
+#define I915_SCHED_DEADLINE_SHIFT 19 /* i.e. roughly 500us buckets */
+#define I915_DEADLINE_NEVER U64_MAX
};
struct i915_dependency {
@@ -81,6 +97,7 @@ struct i915_dependency {
#define I915_DEPENDENCY_ALLOC BIT(0)
#define I915_DEPENDENCY_EXTERNAL BIT(1)
#define I915_DEPENDENCY_WEAK BIT(2)
+ u64 ipi_deadline;
int ipi_priority;
};
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index 1929feba4e8e..29ff6b669cc2 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -24,6 +24,7 @@ selftest(uncore, intel_uncore_mock_selftests)
selftest(engine, intel_engine_cs_mock_selftests)
selftest(timelines, intel_timeline_mock_selftests)
selftest(requests, i915_request_mock_selftests)
+selftest(scheduler, i915_scheduler_mock_selftests)
selftest(objects, i915_gem_object_mock_selftests)
selftest(phys, i915_gem_phys_mock_selftests)
selftest(dmabuf, i915_gem_dmabuf_mock_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 92c628f18c60..db91e639918e 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -2124,6 +2124,7 @@ static int measure_preemption(struct intel_context *ce)
intel_ring_advance(rq, cs);
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
+ rq->sched.deadline = 0;
elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
i915_request_add(rq);
diff --git a/drivers/gpu/drm/i915/selftests/i915_scheduler.c b/drivers/gpu/drm/i915/selftests/i915_scheduler.c
new file mode 100644
index 000000000000..9ca50db81034
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/i915_scheduler.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#include "i915_selftest.h"
+
+static int mock_scheduler_slices(void *dummy)
+{
+ u64 min, max, normal, kernel;
+
+ min = prio_slice(I915_PRIORITY_MIN);
+ pr_info("%8s slice: %lluus\n", "min", min >> 10);
+
+ normal = prio_slice(0);
+ pr_info("%8s slice: %lluus\n", "normal", normal >> 10);
+
+ max = prio_slice(I915_PRIORITY_MAX);
+ pr_info("%8s slice: %lluus\n", "max", max >> 10);
+
+ kernel = prio_slice(I915_PRIORITY_BARRIER);
+ pr_info("%8s slice: %lluus\n", "kernel", kernel >> 10);
+
+ if (kernel != 0) {
+ pr_err("kernel prio slice should be 0\n");
+ return -EINVAL;
+ }
+
+ if (max >= normal) {
+ pr_err("maximum prio slice should be shorter than normal\n");
+ return -EINVAL;
+ }
+
+ if (min <= normal) {
+ pr_err("minimum prio slice should be longer than normal\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int i915_scheduler_mock_selftests(void)
+{
+ static const struct i915_subtest tests[] = {
+ SUBTEST(mock_scheduler_slices),
+ };
+
+ return i915_subtests(tests, NULL);
+}
--
2.20.1
More information about the Intel-gfx-trybot
mailing list