[PATCH 8/8] vdeadline
Chris Wilson
chris at chris-wilson.co.uk
Fri Jun 5 13:17:45 UTC 2020
---
drivers/gpu/drm/i915/display/intel_display.c | 2 +-
drivers/gpu/drm/i915/gem/i915_gem_object.h | 4 +-
drivers/gpu/drm/i915/gem/i915_gem_wait.c | 21 +-
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 13 +-
.../gpu/drm/i915/gt/intel_engine_heartbeat.c | 8 +-
drivers/gpu/drm/i915/gt/intel_engine_pm.c | 2 +-
drivers/gpu/drm/i915/gt/intel_engine_types.h | 14 +-
drivers/gpu/drm/i915/gt/intel_lrc.c | 260 ++++++------------
drivers/gpu/drm/i915/gt/selftest_hangcheck.c | 5 +-
drivers/gpu/drm/i915/gt/selftest_lrc.c | 34 ++-
.../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +-
drivers/gpu/drm/i915/i915_priolist_types.h | 6 +-
drivers/gpu/drm/i915/i915_request.c | 2 +-
drivers/gpu/drm/i915/i915_request.h | 4 +-
drivers/gpu/drm/i915/i915_scheduler.c | 210 ++++++++------
drivers/gpu/drm/i915/i915_scheduler.h | 25 +-
drivers/gpu/drm/i915/i915_scheduler_types.h | 17 +-
17 files changed, 311 insertions(+), 320 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 797e3573d392..5bb20f701a44 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -15964,7 +15964,7 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
if (ret)
return ret;
- i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
+ i915_gem_object_wait_deadline(obj, 0, ktime_get() /* next vblank? */);
i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
if (!new_plane_state->uapi.fence) { /* implicit fencing */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 876c34982555..7bcd2661de4c 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -474,9 +474,9 @@ static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
int i915_gem_object_wait(struct drm_i915_gem_object *obj,
unsigned int flags,
long timeout);
-int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
+int i915_gem_object_wait_deadline(struct drm_i915_gem_object *obj,
unsigned int flags,
- int prio);
+ ktime_t deadline);
void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
enum fb_op_origin origin);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c
index cefbbb3d9b52..5224d4363ea3 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c
@@ -93,17 +93,18 @@ i915_gem_object_wait_reservation(struct dma_resv *resv,
return timeout;
}
-static void __fence_set_priority(struct dma_fence *fence, int prio)
+static void __fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
{
if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
return;
local_bh_disable();
- i915_request_set_priority(to_request(fence), prio);
+ i915_request_set_deadline(to_request(fence),
+ i915_sched_to_ticks(deadline));
local_bh_enable(); /* kick the tasklets if queues were reprioritised */
}
-static void fence_set_priority(struct dma_fence *fence, int prio)
+static void fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
{
/* Recurse once into a fence-array */
if (dma_fence_is_array(fence)) {
@@ -111,16 +112,16 @@ static void fence_set_priority(struct dma_fence *fence, int prio)
int i;
for (i = 0; i < array->num_fences; i++)
- __fence_set_priority(array->fences[i], prio);
+ __fence_set_deadline(array->fences[i], deadline);
} else {
- __fence_set_priority(fence, prio);
+ __fence_set_deadline(fence, deadline);
}
}
int
-i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
+i915_gem_object_wait_deadline(struct drm_i915_gem_object *obj,
unsigned int flags,
- int prio)
+ ktime_t deadline)
{
struct dma_fence *excl;
@@ -130,12 +131,12 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
int ret;
ret = dma_resv_get_fences_rcu(obj->base.resv,
- &excl, &count, &shared);
+ &excl, &count, &shared);
if (ret)
return ret;
for (i = 0; i < count; i++) {
- fence_set_priority(shared[i], prio);
+ fence_set_deadline(shared[i], deadline);
dma_fence_put(shared[i]);
}
@@ -145,7 +146,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
}
if (excl) {
- fence_set_priority(excl, prio);
+ fence_set_deadline(excl, deadline);
dma_fence_put(excl);
}
return 0;
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index b2ebadbc8523..8942cf936111 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -513,7 +513,7 @@ void intel_engine_init_execlists(struct intel_engine_cs *engine)
execlists->active =
memset(execlists->inflight, 0, sizeof(execlists->inflight));
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = KTIME_MAX;
execlists->queue = RB_ROOT_CACHED;
}
@@ -1188,14 +1188,15 @@ bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
}
}
-static int print_sched_attr(const struct i915_sched_attr *attr,
- char *buf, int x, int len)
+static int print_sched(const struct i915_sched_node *node,
+ char *buf, int x, int len)
{
- if (attr->priority == I915_PRIORITY_INVALID)
+ if (node->attr.priority == I915_PRIORITY_INVALID)
return x;
x += snprintf(buf + x, len - x,
- " prio=%d", attr->priority);
+ " prio=%d, deadline=%llu",
+ node->attr.priority, node->deadline);
return x;
}
@@ -1208,7 +1209,7 @@ static void print_request(struct drm_printer *m,
char buf[80] = "";
int x = 0;
- x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
+ x = print_sched(&rq->sched, buf, x, sizeof(buf));
drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
prefix,
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 37bc03e05113..852225ce814b 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -89,6 +89,8 @@ static void heartbeat(struct work_struct *wrk)
*/
} else if (intel_engine_has_scheduler(engine) &&
rq->sched.attr.priority < I915_PRIORITY_BARRIER) {
+ u64 deadline;
+
/*
* Gradually raise the priority of the heartbeat to
* give high priority work [which presumably desires
@@ -100,9 +102,13 @@ static void heartbeat(struct work_struct *wrk)
attr.priority |= I915_PRIORITY_HEARTBEAT;
if (rq->sched.attr.priority >= attr.priority)
attr.priority = I915_PRIORITY_BARRIER;
+ rq->sched.attr.priority = attr.priority;
+
+ deadline = intel_engine_next_virtual_deadline(engine,
+ attr.priority);
local_bh_disable();
- i915_request_set_priority(rq, attr.priority);
+ i915_request_set_deadline(rq, deadline);
local_bh_enable();
} else {
if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
index d0a1078ef632..ad30ec83a74f 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c
@@ -249,7 +249,7 @@ static int __engine_park(struct intel_wakeref *wf)
intel_engine_disarm_breadcrumbs(engine);
/* Must be reset upon idling, or we may miss the busy wakeup. */
- GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
+ GEM_BUG_ON(engine->execlists.queue_deadline_hint != KTIME_MAX);
if (engine->park)
engine->park(engine);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 48e111f16dc5..cc2d95069bf8 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -232,17 +232,7 @@ struct intel_engine_execlists {
unsigned int port_mask;
/**
- * @switch_priority_hint: Second context priority.
- *
- * We submit multiple contexts to the HW simultaneously and would
- * like to occasionally switch between them to emulate timeslicing.
- * To know when timeslicing is suitable, we track the priority of
- * the context submitted second.
- */
- int switch_priority_hint;
-
- /**
- * @queue_priority_hint: Highest pending priority.
+ * @queue_deadline_hint: Next available deadline.
*
* When we add requests into the queue, or adjust the priority of
* executing requests, we compute the maximum priority of those
@@ -253,7 +243,7 @@ struct intel_engine_execlists {
* dequeuing the priority hint may no longer may match the highest
* available request priority.
*/
- int queue_priority_hint;
+ u64 queue_deadline_hint;
/**
* @queue: queue of requests, in priority lists
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 69bd533f0ac9..b131990eddb9 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -200,7 +200,7 @@ struct virtual_engine {
*/
struct ve_node {
struct rb_node rb;
- int prio;
+ u64 deadline;
} nodes[I915_NUM_ENGINES];
/*
@@ -414,9 +414,14 @@ static inline int rq_prio(const struct i915_request *rq)
return READ_ONCE(rq->sched.attr.priority);
}
-static int effective_prio(const struct i915_request *rq)
+static inline u64 rq_deadline(const struct i915_request *rq)
{
- int prio = rq_prio(rq);
+ return READ_ONCE(rq->sched.deadline);
+}
+
+static u64 effective_deadline(const struct i915_request *rq)
+{
+ u64 deadline = rq_deadline(rq);
/*
* If this request is special and must not be interrupted at any
@@ -427,27 +432,27 @@ static int effective_prio(const struct i915_request *rq)
* nopreempt for as long as desired).
*/
if (i915_request_has_nopreempt(rq))
- prio = I915_PRIORITY_UNPREEMPTABLE;
+ deadline = 0;
- return prio;
+ return deadline;
}
-static int queue_prio(const struct intel_engine_execlists *execlists)
+static u64 queue_deadline(const struct intel_engine_execlists *execlists)
{
struct rb_node *rb;
rb = rb_first_cached(&execlists->queue);
if (!rb)
- return INT_MIN;
+ return KTIME_MAX;
- return to_priolist(rb)->priority;
+ return to_priolist(rb)->deadline;
}
static inline bool need_preempt(const struct intel_engine_cs *engine,
const struct i915_request *rq,
struct rb_node *rb)
{
- int last_prio;
+ u64 last_deadline;
if (!intel_engine_has_semaphores(engine))
return false;
@@ -470,8 +475,8 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* priority level: the task that is running should remain running
* to preserve FIFO ordering of dependencies.
*/
- last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
- if (engine->execlists.queue_priority_hint <= last_prio)
+ last_deadline = effective_deadline(rq);
+ if (engine->execlists.queue_deadline_hint >= last_deadline)
return false;
/*
@@ -479,7 +484,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* power of PI, be the highest priority of that context.
*/
if (!list_is_last(&rq->sched.link, &engine->active.requests) &&
- rq_prio(list_next_entry(rq, sched.link)) > last_prio)
+ rq_deadline(list_next_entry(rq, sched.link)) < last_deadline)
return true;
if (rb) {
@@ -493,7 +498,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
rcu_read_lock();
next = READ_ONCE(ve->request);
if (next)
- preempt = rq_prio(next) > last_prio;
+ preempt = rq_deadline(next) < last_deadline;
rcu_read_unlock();
}
@@ -511,7 +516,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
* context, it's priority would not exceed ELSP[0] aka last_prio.
*/
- return queue_prio(&engine->execlists) > last_prio;
+ return queue_deadline(&engine->execlists) < last_deadline;
}
__maybe_unused static inline bool
@@ -528,7 +533,7 @@ assert_priority_queue(const struct i915_request *prev,
if (i915_request_is_active(prev))
return true;
- return rq_prio(prev) >= rq_prio(next);
+ return rq_deadline(prev) <= rq_deadline(next);
}
/*
@@ -1098,7 +1103,7 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
{
struct i915_request *rq, *rn, *active = NULL;
struct list_head *uninitialized_var(pl);
- int prio = I915_PRIORITY_INVALID;
+ u64 deadline = 0;
lockdep_assert_held(&engine->active.lock);
@@ -1119,9 +1124,11 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
*/
if (likely(rq->execution_mask == engine->mask)) {
GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
- if (rq_prio(rq) != prio) {
- prio = rq_prio(rq);
- pl = i915_sched_lookup_priolist(engine, prio);
+ if (rq_deadline(rq) != deadline) {
+ deadline = rq_deadline(rq);
+ pl = i915_sched_lookup_priolist(engine,
+ deadline);
+
}
GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
@@ -1536,14 +1543,14 @@ dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
if (!rq)
return "";
- snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
+ snprintf(buf, buflen, "%sccid:%x %llx:%lld%s dl %llu",
prefix,
rq->context->lrc.ccid,
rq->fence.context, rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
- rq_prio(rq));
+ rq_deadline(rq));
return buf;
}
@@ -1822,7 +1829,9 @@ static void virtual_xfer_breadcrumbs(struct virtual_engine *ve)
intel_engine_transfer_stale_breadcrumbs(ve->siblings[0], &ve->context);
}
-static void defer_request(struct i915_request *rq, struct list_head * const pl)
+static void defer_request(struct i915_request *rq,
+ struct list_head * const pl,
+ u64 deadline)
{
LIST_HEAD(list);
@@ -1837,6 +1846,7 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
struct i915_dependency *p;
GEM_BUG_ON(i915_request_is_active(rq));
+ rq->sched.deadline = deadline;
list_move_tail(&rq->sched.link, pl);
for_each_waiter(p, rq) {
@@ -1859,10 +1869,9 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
if (!i915_request_is_ready(w))
continue;
- if (rq_prio(w) < rq_prio(rq))
+ if (rq_deadline(w) > deadline)
continue;
- GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
list_move_tail(&w->sched.link, &list);
}
@@ -1873,48 +1882,17 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl)
static void defer_active(struct intel_engine_cs *engine)
{
struct i915_request *rq;
+ u64 deadline;
rq = __unwind_incomplete_requests(engine);
if (!rq)
return;
- defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq)));
-}
-
-static bool
-need_timeslice(const struct intel_engine_cs *engine,
- const struct i915_request *rq,
- const struct rb_node *rb)
-{
- int hint;
-
- if (!intel_engine_has_timeslices(engine))
- return false;
-
- hint = engine->execlists.queue_priority_hint;
-
- if (rb) {
- const struct virtual_engine *ve =
- rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
- const struct intel_engine_cs *inflight =
- intel_context_inflight(&ve->context);
-
- if (!inflight || inflight == engine) {
- struct i915_request *next;
-
- rcu_read_lock();
- next = READ_ONCE(ve->request);
- if (next)
- hint = max(hint, rq_prio(next));
- rcu_read_unlock();
- }
- }
-
- if (!list_is_last(&rq->sched.link, &engine->active.requests))
- hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
-
- GEM_BUG_ON(hint >= I915_PRIORITY_UNPREEMPTABLE);
- return hint >= effective_prio(rq);
+ deadline = max(rq_deadline(rq),
+ intel_engine_next_virtual_deadline(engine, rq_prio(rq)));
+ defer_request(rq,
+ i915_sched_lookup_priolist(engine, deadline),
+ deadline);
}
static bool
@@ -1937,23 +1915,17 @@ timeslice_yield(const struct intel_engine_execlists *el,
}
static bool
-timeslice_expired(const struct intel_engine_execlists *el,
- const struct i915_request *rq)
+timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
{
- return timer_expired(&el->timer) || timeslice_yield(el, rq);
-}
+ const struct intel_engine_execlists *el = &engine->execlists;
-static int
-switch_prio(struct intel_engine_cs *engine, const struct i915_request *rq)
-{
- if (list_is_last(&rq->sched.link, &engine->active.requests))
- return INT_MIN;
+ if (!intel_engine_has_timeslices(engine))
+ return false;
- return rq_prio(list_next_entry(rq, sched.link));
+ return timer_expired(&el->timer) || timeslice_yield(el, rq);
}
-static inline unsigned long
-timeslice(const struct intel_engine_cs *engine)
+static unsigned long timeslice(const struct intel_engine_cs *engine)
{
return READ_ONCE(engine->props.timeslice_duration_ms);
}
@@ -1966,9 +1938,6 @@ static unsigned long active_timeslice(const struct intel_engine_cs *engine)
if (!rq || i915_request_completed(rq))
return 0;
- if (READ_ONCE(execlists->switch_priority_hint) < effective_prio(rq))
- return 0;
-
return timeslice(engine);
}
@@ -1985,29 +1954,6 @@ static void set_timeslice(struct intel_engine_cs *engine)
set_timer_ms(&engine->execlists.timer, duration);
}
-static void start_timeslice(struct intel_engine_cs *engine, int prio)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
- unsigned long duration;
-
- if (!intel_engine_has_timeslices(engine))
- return;
-
- WRITE_ONCE(execlists->switch_priority_hint, prio);
- if (prio == INT_MIN)
- return;
-
- if (timer_pending(&execlists->timer))
- return;
-
- duration = timeslice(engine);
- ENGINE_TRACE(engine,
- "start timeslicing, prio:%d, interval:%lu",
- prio, duration);
-
- set_timer_ms(&execlists->timer, duration);
-}
-
static void record_preemption(struct intel_engine_execlists *execlists)
{
(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
@@ -2123,11 +2069,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
}
ENGINE_TRACE(engine,
- "preempting last=%llx:%lld, prio=%d, hint=%d\n",
+ "preempting last=%llx:%llu, deadline=%llu, hint=%llu\n",
last->fence.context,
last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint);
+ rq_deadline(last),
+ execlists->queue_deadline_hint);
record_preemption(execlists);
/*
@@ -2147,19 +2093,18 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
__unwind_incomplete_requests(engine);
last = NULL;
- } else if (need_timeslice(engine, last, rb) &&
- timeslice_expired(execlists, last)) {
+ } else if (timeslice_expired(engine, last)) {
if (i915_request_completed(last)) {
tasklet_hi_schedule(&execlists->tasklet);
return;
}
ENGINE_TRACE(engine,
- "expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
+ "expired last=%llx:%llu, deadline=%llu, hint=%llu, yield?=%s\n",
last->fence.context,
last->fence.seqno,
- last->sched.attr.priority,
- execlists->queue_priority_hint,
+ rq_deadline(last),
+ execlists->queue_deadline_hint,
yesno(timeslice_yield(execlists, last)));
ring_set_paused(engine, 1);
@@ -2195,7 +2140,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* Even if ELSP[1] is occupied and not worthy
* of timeslices, our queue might be.
*/
- start_timeslice(engine, queue_prio(execlists));
return;
}
}
@@ -2221,7 +2165,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
GEM_BUG_ON(rq->engine != &ve->base);
GEM_BUG_ON(rq->context != &ve->context);
- if (rq_prio(rq) >= queue_prio(execlists)) {
+ if (rq_deadline(rq) < queue_deadline(execlists)) {
if (!virtual_matches(ve, rq, engine)) {
spin_unlock(&ve->base.active.lock);
rb = rb_next(rb);
@@ -2230,7 +2174,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (last && !can_merge_rq(last, rq)) {
spin_unlock(&ve->base.active.lock);
- start_timeslice(engine, rq_prio(rq));
return; /* leave this for another sibling */
}
@@ -2244,8 +2187,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
yesno(engine != ve->siblings[0]));
WRITE_ONCE(ve->request, NULL);
- WRITE_ONCE(ve->base.execlists.queue_priority_hint,
- INT_MIN);
+ WRITE_ONCE(ve->base.execlists.queue_deadline_hint,
+ KTIME_MAX);
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
@@ -2342,10 +2285,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (last->context == rq->context)
goto done;
- if (i915_request_has_sentinel(last)) {
- start_timeslice(engine, rq_prio(rq));
+ if (i915_request_has_sentinel(last))
goto done;
- }
/*
* If GVT overrides us we only ever submit
@@ -2401,12 +2342,10 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
* request triggering preemption on the next dequeue (or subsequent
* interrupt for secondary ports).
*/
- execlists->queue_priority_hint = queue_prio(execlists);
+ execlists->queue_deadline_hint = queue_deadline(execlists);
if (submit) {
*port = execlists_schedule_in(last, port - execlists->pending);
- execlists->switch_priority_hint =
- switch_prio(engine, *execlists->pending);
/*
* Skip if we ended up with exactly the same set of requests,
@@ -2426,6 +2365,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
set_preempt_timeout(engine, *active);
execlists_submit_ports(engine);
} else {
+ set_timeslice(engine);
skip_submit:
ring_set_paused(engine, 0);
}
@@ -2817,9 +2757,10 @@ static bool hold_request(const struct i915_request *rq)
return result;
}
-static void __execlists_unhold(struct i915_request *rq)
+static bool __execlists_unhold(struct i915_request *rq)
{
LIST_HEAD(list);
+ bool submit = false;
do {
struct i915_dependency *p;
@@ -2830,10 +2771,7 @@ static void __execlists_unhold(struct i915_request *rq)
GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
i915_request_clear_hold(rq);
- list_move_tail(&rq->sched.link,
- i915_sched_lookup_priolist(rq->engine,
- rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ submit |= intel_engine_queue_request(rq->engine, rq);
/* Also release any children on this engine that are ready */
for_each_waiter(p, rq) {
@@ -2862,6 +2800,8 @@ static void __execlists_unhold(struct i915_request *rq)
rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
} while (rq);
+
+ return submit;
}
static void execlists_unhold(struct intel_engine_cs *engine,
@@ -2873,12 +2813,8 @@ static void execlists_unhold(struct intel_engine_cs *engine,
* Move this request back to the priority queue, and all of its
* children and grandchildren that were suspended along with it.
*/
- __execlists_unhold(rq);
-
- if (rq_prio(rq) > engine->execlists.queue_priority_hint) {
- engine->execlists.queue_priority_hint = rq_prio(rq);
+ if (__execlists_unhold(rq))
tasklet_hi_schedule(&engine->execlists.tasklet);
- }
spin_unlock_irq(&engine->active.lock);
}
@@ -3141,15 +3077,6 @@ static void execlists_preempt(struct timer_list *timer)
execlists_kick(timer, preempt);
}
-static void queue_request(struct intel_engine_cs *engine,
- struct i915_request *rq)
-{
- GEM_BUG_ON(!list_empty(&rq->sched.link));
- list_add_tail(&rq->sched.link,
- i915_sched_lookup_priolist(engine, rq_prio(rq)));
- set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
-}
-
static void __submit_queue_imm(struct intel_engine_cs *engine)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -3167,18 +3094,6 @@ static void __submit_queue_imm(struct intel_engine_cs *engine)
__execlists_submission_tasklet(engine);
}
-static void submit_queue(struct intel_engine_cs *engine,
- const struct i915_request *rq)
-{
- struct intel_engine_execlists *execlists = &engine->execlists;
-
- if (rq_prio(rq) <= execlists->queue_priority_hint)
- return;
-
- execlists->queue_priority_hint = rq_prio(rq);
- __submit_queue_imm(engine);
-}
-
static bool ancestor_on_hold(const struct intel_engine_cs *engine,
const struct i915_request *rq)
{
@@ -3199,12 +3114,8 @@ static void execlists_submit_request(struct i915_request *request)
list_add_tail(&request->sched.link, &engine->active.hold);
i915_request_set_hold(request);
} else {
- queue_request(engine, request);
-
- GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
- GEM_BUG_ON(list_empty(&request->sched.link));
-
- submit_queue(engine, request);
+ if (intel_engine_queue_request(engine, request))
+ __submit_queue_imm(engine);
}
spin_unlock_irqrestore(&engine->active.lock, flags);
@@ -4262,7 +4173,7 @@ static void nop_submission_tasklet(unsigned long data)
struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
/* The driver is wedged; don't process any more events. */
- WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN);
+ WRITE_ONCE(engine->execlists.queue_deadline_hint, KTIME_MAX);
}
static void execlists_reset_cancel(struct intel_engine_cs *engine)
@@ -4330,14 +4241,14 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine)
__i915_request_submit(rq);
i915_request_put(rq);
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = KTIME_MAX;
}
spin_unlock(&ve->base.active.lock);
}
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = KTIME_MAX;
execlists->queue = RB_ROOT_CACHED;
GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
@@ -5452,9 +5363,9 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
mask = ve->siblings[0]->mask;
}
- ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
+ ENGINE_TRACE(&ve->base, "rq=%llx:%llu, mask=%x, hint=%llu\n",
rq->fence.context, rq->fence.seqno,
- mask, ve->base.execlists.queue_priority_hint);
+ mask, ve->base.execlists.queue_deadline_hint);
return mask;
}
@@ -5462,7 +5373,7 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
static void virtual_submission_tasklet(unsigned long data)
{
struct virtual_engine * const ve = (struct virtual_engine *)data;
- const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint);
+ const u64 deadline = READ_ONCE(ve->base.execlists.queue_deadline_hint);
intel_engine_mask_t mask;
unsigned int n;
@@ -5502,7 +5413,8 @@ static void virtual_submission_tasklet(unsigned long data)
*/
first = rb_first_cached(&sibling->execlists.virtual) ==
&node->rb;
- if (prio == node->prio || (prio > node->prio && first))
+ if (deadline == node->deadline ||
+ (deadline < node->deadline && first))
goto submit_engine;
rb_erase_cached(&node->rb, &sibling->execlists.virtual);
@@ -5516,7 +5428,7 @@ static void virtual_submission_tasklet(unsigned long data)
rb = *parent;
other = rb_entry(rb, typeof(*other), rb);
- if (prio > other->prio) {
+ if (deadline < other->deadline) {
parent = &rb->rb_left;
} else {
parent = &rb->rb_right;
@@ -5531,8 +5443,8 @@ static void virtual_submission_tasklet(unsigned long data)
submit_engine:
GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
- node->prio = prio;
- if (first && prio > sibling->execlists.queue_priority_hint)
+ node->deadline = deadline;
+ if (first && deadline < sibling->execlists.queue_deadline_hint)
tasklet_hi_schedule(&sibling->execlists.tasklet);
spin_unlock(&sibling->active.lock);
@@ -5564,10 +5476,13 @@ static void virtual_submit_request(struct i915_request *rq)
if (i915_request_completed(rq)) {
__i915_request_submit(rq);
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = KTIME_MAX;
ve->request = NULL;
} else {
- ve->base.execlists.queue_priority_hint = rq_prio(rq);
+ rq->sched.deadline =
+ intel_engine_next_virtual_deadline(&ve->base,
+ rq_prio(rq));
+ ve->base.execlists.queue_deadline_hint = rq_deadline(rq);
ve->request = i915_request_get(rq);
GEM_BUG_ON(!list_empty(virtual_queue(ve)));
@@ -5671,7 +5586,7 @@ intel_execlists_create_virtual(struct intel_engine_cs **siblings,
ve->base.bond_execute = virtual_bond_execute;
INIT_LIST_HEAD(virtual_queue(ve));
- ve->base.execlists.queue_priority_hint = INT_MIN;
+ ve->base.execlists.queue_deadline_hint = KTIME_MAX;
tasklet_init(&ve->base.execlists.tasklet,
virtual_submission_tasklet,
(unsigned long)ve);
@@ -5858,12 +5773,9 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine,
show_request(m, last, "\t\tE ");
}
- if (execlists->switch_priority_hint != INT_MIN)
- drm_printf(m, "\t\tSwitch priority hint: %d\n",
- READ_ONCE(execlists->switch_priority_hint));
- if (execlists->queue_priority_hint != INT_MIN)
- drm_printf(m, "\t\tQueue priority hint: %d\n",
- READ_ONCE(execlists->queue_priority_hint));
+ if (execlists->queue_deadline_hint != KTIME_MAX)
+ drm_printf(m, "\t\tQueue deadline hint: %llu\n",
+ READ_ONCE(execlists->queue_deadline_hint));
last = NULL;
count = 0;
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index fe5a5bb07fe8..7e3eb3738c17 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -730,8 +730,11 @@ static int active_engine(void *data)
if (intel_engine_has_scheduler(engine) &&
arg->flags & TEST_PRIORITY) {
int prio = i915_prandom_u32_max_state(512, &prng);
+ u64 deadline =
+ intel_engine_next_virtual_deadline(engine,
+ prio);
- i915_request_set_priority(rq[idx], prio);
+ i915_request_set_deadline(rq[idx], deadline);
}
err = active_request_put(old);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 5cfadab091ea..e8fd1a46fc7c 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -308,8 +308,11 @@ static int live_unlite_restore(struct intel_gt *gt, int prio)
i915_request_put(rq[0]);
if (prio) {
+ u64 deadline =
+ intel_engine_next_virtual_deadline(rq[1]->engine,
+ prio);
/* Alternatively preempt the spinner with ce[1] */
- i915_request_set_priority(rq[1], prio);
+ i915_request_set_deadline(rq[1], deadline);
}
/* And switch back to ce[0] for good measure */
@@ -754,7 +757,7 @@ semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
static int
release_queue(struct intel_engine_cs *engine,
struct i915_vma *vma,
- int idx, int prio)
+ int idx, u64 deadline)
{
struct i915_request *rq;
u32 *cs;
@@ -780,7 +783,7 @@ release_queue(struct intel_engine_cs *engine,
i915_request_add(rq);
local_bh_disable();
- i915_request_set_priority(rq, prio);
+ i915_request_set_deadline(rq, deadline);
local_bh_enable(); /* kick tasklet */
i915_request_put(rq);
@@ -816,7 +819,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
}
}
- err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
+ err = release_queue(outer, vma, n, 0);
if (err)
goto out;
@@ -1204,7 +1207,7 @@ static int live_timeslice_queue(void *arg)
err = PTR_ERR(rq);
goto err_heartbeat;
}
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
err = wait_for_submit(engine, rq, HZ / 2);
if (err) {
pr_err("%s: Timed out trying to submit semaphores\n",
@@ -1230,7 +1233,7 @@ static int live_timeslice_queue(void *arg)
GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
/* Queue: semaphore signal, matching priority as semaphore */
- err = release_queue(engine, vma, 1, effective_prio(rq));
+ err = release_queue(engine, vma, 1, effective_deadline(rq));
if (err)
goto err_rq;
@@ -1769,7 +1772,7 @@ static int live_late_preempt(void *arg)
goto err_wedged;
}
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
if (!igt_wait_for_spinner(&spin_hi, rq)) {
pr_err("High priority context failed to preempt the low priority context\n");
@@ -2315,7 +2318,7 @@ static int live_suppress_self_preempt(void *arg)
i915_request_add(rq_b);
GEM_BUG_ON(i915_request_completed(rq_a));
- i915_request_set_priority(rq_a, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq_a, 0);
igt_spinner_end(&a.spin);
if (!igt_wait_for_spinner(&b.spin, rq_b)) {
@@ -2626,7 +2629,7 @@ static int live_chain_preempt(void *arg)
i915_request_get(rq);
i915_request_add(rq);
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
igt_spinner_end(&hi.spin);
if (i915_request_wait(rq, 0, HZ / 5) < 0) {
@@ -2815,12 +2818,17 @@ static int live_preempt_gang(void *arg)
return -EIO;
do {
+ u64 deadline;
+
err = create_gang(engine, &rq);
if (err)
break;
/* Submit each spinner at increasing priority */
- i915_request_set_priority(rq, prio++);
+ deadline =
+ intel_engine_next_virtual_deadline(engine,
+ prio++);
+ i915_request_set_deadline(rq, deadline);
if (prio < 0)
break;
@@ -3073,7 +3081,7 @@ static int preempt_user(struct intel_engine_cs *engine,
i915_request_get(rq);
i915_request_add(rq);
- i915_request_set_priority(rq, I915_PRIORITY_MAX);
+ i915_request_set_deadline(rq, 0);
if (i915_request_wait(rq, 0, HZ / 2) < 0)
err = -ETIME;
@@ -4607,7 +4615,7 @@ static int emit_semaphore_signal(struct intel_context *ce, void *slot)
intel_ring_advance(rq, cs);
- rq->sched.attr.priority = I915_PRIORITY_BARRIER;
+ rq->sched.deadline = 0;
i915_request_add(rq);
return 0;
}
@@ -5673,7 +5681,7 @@ static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
intel_ring_advance(rq, cs);
- rq->sched.attr.priority = I915_PRIORITY_BARRIER;
+ rq->sched.deadline = 0;
err_rq:
i915_request_add(rq);
err_batch:
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 0c42e8b0c211..b2f70fae80e5 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -333,8 +333,6 @@ static void __guc_dequeue(struct intel_engine_cs *engine)
i915_priolist_free(p);
}
done:
- execlists->queue_priority_hint =
- rb ? to_priolist(rb)->priority : INT_MIN;
if (submit) {
*port = schedule_in(last, port - execlists->inflight);
*++port = NULL;
@@ -476,7 +474,7 @@ static void guc_reset_cancel(struct intel_engine_cs *engine)
/* Remaining _unready_ requests will be nop'ed when submitted */
- execlists->queue_priority_hint = INT_MIN;
+ execlists->queue_deadline_hint = KTIME_MAX;
execlists->queue = RB_ROOT_CACHED;
spin_unlock_irqrestore(&engine->active.lock, flags);
diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h
index bc2fa84f98a8..6aac9aa31773 100644
--- a/drivers/gpu/drm/i915/i915_priolist_types.h
+++ b/drivers/gpu/drm/i915/i915_priolist_types.h
@@ -8,6 +8,7 @@
#define _I915_PRIOLIST_TYPES_H_
#include <linux/list.h>
+#include <linux/ktime.h>
#include <linux/rbtree.h>
#include <uapi/drm/i915_drm.h>
@@ -35,13 +36,12 @@ enum {
* i.e. nothing can have higher priority and force us to usurp the
* active request.
*/
-#define I915_PRIORITY_UNPREEMPTABLE INT_MAX
-#define I915_PRIORITY_BARRIER (I915_PRIORITY_UNPREEMPTABLE - 1)
+#define I915_PRIORITY_BARRIER INT_MAX
struct i915_priolist {
struct list_head requests;
struct rb_node node;
- int priority;
+ u64 deadline;
};
#endif /* _I915_PRIOLIST_TYPES_H_ */
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index bd7a7e8e0f13..35107b8162ad 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -1575,7 +1575,7 @@ void __i915_request_queue(struct i915_request *rq,
* run at the earliest possible convenience.
*/
if (attr)
- i915_request_set_priority(rq, attr->priority);
+ rq->sched.attr = *attr;
i915_sw_fence_commit(&rq->semaphore);
i915_sw_fence_commit(&rq->submit);
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 118ab6650d1f..23594e712292 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -561,7 +561,7 @@ static inline void i915_request_clear_hold(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_timeline(struct i915_request *rq)
+i915_request_timeline(const struct i915_request *rq)
{
/* Valid only while the request is being constructed (or retired). */
return rcu_dereference_protected(rq->timeline,
@@ -576,7 +576,7 @@ i915_request_gem_context(struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_active_timeline(struct i915_request *rq)
+i915_request_active_timeline(const struct i915_request *rq)
{
/*
* When in use during submission, we are protected by a guarantee that
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index d837c038b53f..d2077521ea44 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -20,7 +20,12 @@ static struct i915_global_scheduler {
static DEFINE_SPINLOCK(ipi_lock);
static LIST_HEAD(ipi_list);
-static inline int rq_prio(const struct i915_request *rq)
+static inline u64 rq_deadline(const struct i915_request *rq)
+{
+ return READ_ONCE(rq->sched.deadline);
+}
+
+static inline u64 rq_prio(const struct i915_request *rq)
{
return READ_ONCE(rq->sched.attr.priority);
}
@@ -32,22 +37,22 @@ static void ipi_schedule(struct irq_work *wrk)
struct i915_dependency *p;
struct i915_request *rq;
unsigned long flags;
- int prio;
+ u64 deadline;
spin_lock_irqsave(&ipi_lock, flags);
p = list_first_entry_or_null(&ipi_list, typeof(*p), ipi_link);
if (p) {
rq = container_of(p->signaler, typeof(*rq), sched);
list_del_init(&p->ipi_link);
- prio = p->ipi_prio;
- p->ipi_prio = INT_MIN;
+ deadline = p->ipi_deadline;
+ p->ipi_deadline = KTIME_MAX;
}
spin_unlock_irqrestore(&ipi_lock, flags);
if (!p)
break;
- if (prio > rq_prio(rq) && !i915_request_completed(rq))
- i915_request_set_priority(rq, prio);
+ if (deadline < rq_deadline(rq) && !i915_request_completed(rq))
+ i915_request_set_deadline(rq, deadline);
} while (1);
rcu_read_unlock();
}
@@ -75,28 +80,8 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb)
return rb_entry(rb, struct i915_priolist, node);
}
-static void assert_priolists(struct intel_engine_execlists * const execlists)
-{
- struct rb_node *rb;
- long last_prio;
-
- if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
- return;
-
- GEM_BUG_ON(rb_first_cached(&execlists->queue) !=
- rb_first(&execlists->queue.rb_root));
-
- last_prio = INT_MAX;
- for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
- const struct i915_priolist *p = to_priolist(rb);
-
- GEM_BUG_ON(p->priority > last_prio);
- last_prio = p->priority;
- }
-}
-
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline)
{
struct intel_engine_execlists * const execlists = &engine->execlists;
struct i915_priolist *p;
@@ -104,10 +89,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
bool first = true;
lockdep_assert_held(&engine->active.lock);
- assert_priolists(execlists);
if (unlikely(execlists->no_priolist))
- prio = I915_PRIORITY_NORMAL;
+ deadline = 0;
find_priolist:
/* most positive priority is scheduled first, equal priorities fifo */
@@ -116,9 +100,9 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
while (*parent) {
rb = *parent;
p = to_priolist(rb);
- if (prio > p->priority) {
+ if (deadline < p->deadline) {
parent = &rb->rb_left;
- } else if (prio < p->priority) {
+ } else if (deadline > p->deadline) {
parent = &rb->rb_right;
first = false;
} else {
@@ -126,13 +110,13 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- if (prio == I915_PRIORITY_NORMAL) {
+ if (!deadline) {
p = &execlists->default_priolist;
} else {
p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC);
/* Convert an allocation failure to a priority bump */
if (unlikely(!p)) {
- prio = I915_PRIORITY_NORMAL; /* recurses just once */
+ deadline = 0; /* recurses just once */
/* To maintain ordering with all rendering, after an
* allocation failure we have to disable all scheduling.
@@ -147,7 +131,7 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
}
}
- p->priority = prio;
+ p->deadline = deadline;
INIT_LIST_HEAD(&p->requests);
rb_link_node(&p->node, rb, parent);
@@ -156,70 +140,54 @@ i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio)
return &p->requests;
}
-void __i915_priolist_free(struct i915_priolist *p)
+void i915_priolist_free(struct i915_priolist *p)
{
- kmem_cache_free(global.slab_priorities, p);
+ if (p->deadline)
+ kmem_cache_free(global.slab_priorities, p);
}
-static inline bool need_preempt(int prio, int active)
-{
- /*
- * Allow preemption of low -> normal -> high, but we do
- * not allow low priority tasks to preempt other low priority
- * tasks under the impression that latency for low priority
- * tasks does not matter (as much as background throughput),
- * so kiss.
- */
- return prio >= max(I915_PRIORITY_NORMAL, active);
-}
-
-static void kick_submission(struct intel_engine_cs *engine,
+static bool kick_submission(struct intel_engine_cs *engine,
const struct i915_request *rq,
- int prio)
+ u64 deadline)
{
const struct i915_request *inflight;
-
- /*
- * We only need to kick the tasklet once for the high priority
- * new context we add into the queue.
- */
- if (prio <= engine->execlists.queue_priority_hint)
- return;
-
- rcu_read_lock();
+ u64 preempt = KTIME_MAX;
/* Nothing currently active? We're overdue for a submission! */
inflight = execlists_active(&engine->execlists);
if (!inflight)
- goto unlock;
+ goto out;
/*
* If we are already the currently executing context, don't
* bother evaluating if we should preempt ourselves.
*/
if (inflight->context == rq->context)
- goto unlock;
+ return false;
+ preempt = rq_deadline(inflight);
ENGINE_TRACE(engine,
- "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
- prio,
+ "bumping queue-deadline-hint:%llu for rq:%llx:%lld, inflight:%llx:%lld deadline %llu\n",
+ deadline,
rq->fence.context, rq->fence.seqno,
inflight->fence.context, inflight->fence.seqno,
- inflight->sched.attr.priority);
+ preempt);
- engine->execlists.queue_priority_hint = prio;
- if (need_preempt(prio, rq_prio(inflight)))
- tasklet_hi_schedule(&engine->execlists.tasklet);
-
-unlock:
- rcu_read_unlock();
+ /*
+ * We only need to kick the tasklet once for the high priority
+ * new context we add into the queue.
+ */
+out:
+ engine->execlists.queue_deadline_hint = deadline;
+ return deadline < preempt;
}
-static void __i915_request_set_priority(struct i915_request *rq, int prio)
+static bool __i915_request_set_deadline(struct i915_request *rq, u64 deadline)
{
struct intel_engine_cs *engine = rq->engine;
struct i915_request *rn;
struct list_head *plist;
+ bool kick = false;
LIST_HEAD(dfs);
lockdep_assert_held(&engine->active.lock);
@@ -251,7 +219,7 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
struct i915_request *s =
container_of(p->signaler, typeof(*s), sched);
- if (rq_prio(s) >= prio)
+ if (rq_deadline(s) <= deadline)
continue;
if (i915_request_completed(s))
@@ -259,8 +227,8 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
if (s->engine != rq->engine) {
spin_lock(&ipi_lock);
- if (prio > p->ipi_prio) {
- p->ipi_prio = prio;
+ if (deadline < p->ipi_deadline) {
+ p->ipi_deadline = deadline;
list_move(&p->ipi_link, &ipi_list);
irq_work_queue(&ipi_work);
}
@@ -272,12 +240,14 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
}
}
- plist = i915_sched_lookup_priolist(engine, prio);
+ plist = i915_sched_lookup_priolist(engine, deadline);
/* Fifo and depth-first replacement ensure our deps execute first */
list_for_each_entry_safe_reverse(rq, rn, &dfs, sched.dfs) {
+ GEM_BUG_ON(deadline > rq_deadline(rq));
+
INIT_LIST_HEAD(&rq->sched.dfs);
- WRITE_ONCE(rq->sched.attr.priority, prio);
+ WRITE_ONCE(rq->sched.deadline, deadline);
/*
* Once the request is ready, it will be placed into the
@@ -287,18 +257,23 @@ static void __i915_request_set_priority(struct i915_request *rq, int prio)
* any preemption required, be dealt with upon submission.
* See engine->submit_request()
*/
- if (!i915_request_is_ready(rq))
- continue;
if (i915_request_in_priority_queue(rq))
list_move_tail(&rq->sched.link, plist);
/* Defer (tasklet) submission until after all updates. */
- kick_submission(engine, rq, prio);
+ if (i915_request_is_ready(rq) &&
+ deadline < engine->execlists.queue_deadline_hint) {
+ rcu_read_lock();
+ kick |= kick_submission(engine, rq, deadline);
+ rcu_read_unlock();
+ }
}
+
+ return kick;
}
-void i915_request_set_priority(struct i915_request *rq, int prio)
+void i915_request_set_deadline(struct i915_request *rq, u64 deadline)
{
struct intel_engine_cs *engine = READ_ONCE(rq->engine);
unsigned long flags;
@@ -322,15 +297,83 @@ void i915_request_set_priority(struct i915_request *rq, int prio)
if (i915_request_completed(rq))
goto unlock;
- if (prio <= rq_prio(rq))
+ if (deadline >= rq_deadline(rq))
goto unlock;
- __i915_request_set_priority(rq, prio);
+ if (__i915_request_set_deadline(rq, deadline))
+ tasklet_hi_schedule(&engine->execlists.tasklet);
unlock:
spin_unlock_irqrestore(&engine->active.lock, flags);
}
+static inline unsigned long timeslice(const struct intel_engine_cs *engine)
+{
+ return READ_ONCE(engine->props.timeslice_duration_ms);
+}
+
+static unsigned int prio_ratio(int prio)
+{
+ return INT_MAX - prio;
+}
+
+static u64 prio_slice(const struct intel_engine_cs *engine, int prio)
+{
+ return timeslice(engine) * prio_ratio(prio);
+}
+
+u64 intel_engine_virtual_deadline(const struct intel_engine_cs *engine,
+ u64 kt, int priority)
+{
+ return i915_sched_to_ticks(kt + prio_slice(engine, priority));
+}
+
+u64 intel_engine_next_virtual_deadline(const struct intel_engine_cs *engine,
+ int priority)
+{
+ return intel_engine_virtual_deadline(engine, ktime_get(), priority);
+}
+
+static inline u64 i915_request_previous_deadline(const struct i915_request *rq)
+{
+ const struct intel_timeline *tl = i915_request_active_timeline(rq);
+ const struct list_head *pos;
+ u64 last = 0;
+
+ rcu_read_lock();
+ pos = READ_ONCE(rq->link.prev);
+ if (pos != &tl->requests) {
+ const struct i915_request *prev =
+ list_entry(pos, typeof(*prev), link);
+
+ if (!i915_request_completed(prev))
+ last = i915_sched_to_ns(rq_deadline(prev));
+ }
+ rcu_read_unlock();
+
+ return last;
+}
+
+static u64 submit_deadline(const struct intel_engine_cs *engine,
+ const struct i915_request *rq)
+{
+ u64 deadline =
+ max_t(u64,
+ i915_request_previous_deadline(rq),
+ ktime_to_ns(ktime_get()));
+
+ return intel_engine_virtual_deadline(engine, deadline, rq_prio(rq));
+}
+
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq)
+{
+ set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
+ return __i915_request_set_deadline(rq,
+ min(submit_deadline(engine, rq),
+ rq->sched.deadline));
+}
+
void i915_sched_node_init(struct i915_sched_node *node)
{
spin_lock_init(&node->lock);
@@ -346,6 +389,7 @@ void i915_sched_node_init(struct i915_sched_node *node)
void i915_sched_node_reinit(struct i915_sched_node *node)
{
node->attr.priority = I915_PRIORITY_INVALID;
+ node->deadline = KTIME_MAX;
node->semaphores = 0;
node->flags = 0;
@@ -378,7 +422,7 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
if (!node_signaled(signal)) {
INIT_LIST_HEAD(&dep->ipi_link);
- dep->ipi_prio = INT_MIN;
+ dep->ipi_deadline = KTIME_MAX;
dep->signaler = signal;
dep->waiter = node;
dep->flags = flags;
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index 53ac819cc786..cb8dcaea2a80 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -33,16 +33,29 @@ int i915_sched_node_add_dependency(struct i915_sched_node *node,
void i915_sched_node_retire(struct i915_sched_node *node);
-void i915_request_set_priority(struct i915_request *request, int prio);
+void i915_request_set_deadline(struct i915_request *request, u64 deadline);
+
+u64 intel_engine_virtual_deadline(const struct intel_engine_cs *engine,
+ u64 kt, int priority);
+u64 intel_engine_next_virtual_deadline(const struct intel_engine_cs *engine,
+ int priority);
+
+bool intel_engine_queue_request(struct intel_engine_cs *engine,
+ struct i915_request *rq);
struct list_head *
-i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio);
+i915_sched_lookup_priolist(struct intel_engine_cs *engine, u64 deadline);
+
+void i915_priolist_free(struct i915_priolist *p);
+
+static inline u64 i915_sched_to_ticks(ktime_t kt)
+{
+ return ktime_to_ns(kt) >> I915_SCHED_DEADLINE_SHIFT;
+}
-void __i915_priolist_free(struct i915_priolist *p);
-static inline void i915_priolist_free(struct i915_priolist *p)
+static inline u64 i915_sched_to_ns(u64 deadline)
{
- if (p->priority != I915_PRIORITY_NORMAL)
- __i915_priolist_free(p);
+ return deadline << I915_SCHED_DEADLINE_SHIFT;
}
#endif /* _I915_SCHEDULER_H_ */
diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h
index 4e02b5c932b1..3684d14b1cdb 100644
--- a/drivers/gpu/drm/i915/i915_scheduler_types.h
+++ b/drivers/gpu/drm/i915/i915_scheduler_types.h
@@ -69,6 +69,21 @@ struct i915_sched_node {
unsigned int flags;
#define I915_SCHED_HAS_EXTERNAL_CHAIN BIT(0)
intel_engine_mask_t semaphores;
+
+ /**
+ * @deadline: [virtual] deadline
+ *
+ * When the request is ready for execution, it is given a quota
+ * (the engine's timeslice) and a virtual deadline. The virtual
+ * deadline is derived from the current time:
+ * ktime_get() + (prio_ratio * timeslice)
+ *
+ * Requests are then executed in order of deadline completion.
+ * Requests with earlier deadlines than currently executing on
+ * the engine will preempt the active requests.
+ */
+ u64 deadline;
+#define I915_SCHED_DEADLINE_SHIFT 20 /* i.e. roughly millisecond buckets */
};
struct i915_dependency {
@@ -81,7 +96,7 @@ struct i915_dependency {
#define I915_DEPENDENCY_ALLOC BIT(0)
#define I915_DEPENDENCY_EXTERNAL BIT(1)
#define I915_DEPENDENCY_WEAK BIT(2)
- int ipi_prio;
+ u64 ipi_deadline;
};
#define for_each_waiter(p__, rq__) \
--
2.20.1
More information about the Intel-gfx-trybot
mailing list