[PATCH 74/75] no-rq-ring
Chris Wilson
chris at chris-wilson.co.uk
Tue Feb 2 00:36:40 UTC 2021
---
.../gpu/drm/i915/gem/i915_gem_execbuffer.c | 4 +-
drivers/gpu/drm/i915/gt/gen2_engine_cs.c | 12 ++---
drivers/gpu/drm/i915/gt/gen6_engine_cs.c | 16 +++----
drivers/gpu/drm/i915/gt/gen8_engine_cs.c | 19 ++++----
drivers/gpu/drm/i915/gt/intel_context.c | 2 +-
drivers/gpu/drm/i915/gt/intel_engine_cs.c | 40 ++++-------------
.../drm/i915/gt/intel_execlists_submission.c | 14 +++---
drivers/gpu/drm/i915/gt/intel_ring.c | 11 ++---
drivers/gpu/drm/i915/gt/intel_ring.h | 10 +++--
.../gpu/drm/i915/gt/intel_ring_scheduler.c | 2 +-
.../gpu/drm/i915/gt/intel_ring_submission.c | 6 +--
drivers/gpu/drm/i915/gt/selftest_context.c | 2 +-
drivers/gpu/drm/i915/gt/selftest_engine_cs.c | 2 +-
drivers/gpu/drm/i915/gt/selftest_execlists.c | 4 +-
drivers/gpu/drm/i915/gt/selftest_timeline.c | 10 ++---
drivers/gpu/drm/i915/i915_active.c | 2 +-
drivers/gpu/drm/i915/i915_active.h | 4 +-
drivers/gpu/drm/i915/i915_gpu_error.c | 2 +-
drivers/gpu/drm/i915/i915_request.c | 44 +++++++++----------
drivers/gpu/drm/i915/i915_request.h | 29 ++++--------
drivers/gpu/drm/i915/i915_scheduler.c | 34 +++++++-------
drivers/gpu/drm/i915/selftests/i915_request.c | 7 +--
22 files changed, 121 insertions(+), 155 deletions(-)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 052ae889bfde..e20d2bfdf41f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -2628,7 +2628,7 @@ static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel
* drains before we can submit our next request.
*/
list_for_each_entry(rq, &tl->requests, link) {
- if (rq->ring != ring)
+ if (i915_request_get_ring(rq) != ring)
continue;
if (__intel_ring_space(rq->postfix,
@@ -3097,7 +3097,7 @@ static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
static int eb_request_add(struct i915_execbuffer *eb, int err)
{
struct i915_request *rq = eb->request;
- struct intel_timeline * const tl = i915_request_timeline(rq);
+ struct intel_timeline * const tl = i915_request_get_timeline(rq);
struct i915_sched_attr attr = {};
struct i915_request *prev;
diff --git a/drivers/gpu/drm/i915/gt/gen2_engine_cs.c b/drivers/gpu/drm/i915/gt/gen2_engine_cs.c
index 4da0a63c231d..5e499e47079d 100644
--- a/drivers/gpu/drm/i915/gt/gen2_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/gen2_engine_cs.c
@@ -149,8 +149,8 @@ static u32 *__gen2_emit_breadcrumb(const struct intel_engine_cs *engine,
struct i915_request *rq, u32 *cs,
int flush, int post)
{
- GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != engine->status_page.vma);
- GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
+ GEM_BUG_ON(i915_request_get_timeline(rq)->hwsp_ggtt != engine->status_page.vma);
+ GEM_BUG_ON(offset_in_page(i915_request_get_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
*cs++ = MI_FLUSH;
@@ -169,7 +169,7 @@ static u32 *__gen2_emit_breadcrumb(const struct intel_engine_cs *engine,
*cs++ = MI_USER_INTERRUPT;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
@@ -184,7 +184,7 @@ u32 *gen3_emit_breadcrumb(const struct intel_engine_cs *engine,
static u32 *__gen4_emit_breadcrumb(struct i915_request *rq, u32 *cs,
int flush, int post)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
GEM_BUG_ON(tl->mode == INTEL_TIMELINE_RELATIVE_CONTEXT);
@@ -217,7 +217,7 @@ static u32 *__gen4_emit_breadcrumb(struct i915_request *rq, u32 *cs,
*cs++ = MI_USER_INTERRUPT;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
@@ -232,7 +232,7 @@ u32 *gen4_emit_breadcrumb_xcs(const struct intel_engine_cs *engine,
int gen4_emit_init_breadcrumb_xcs(const struct intel_engine_cs *engine,
struct i915_request *rq)
{
- struct intel_timeline *tl = i915_request_timeline(rq);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 *cs;
GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq));
diff --git a/drivers/gpu/drm/i915/gt/gen6_engine_cs.c b/drivers/gpu/drm/i915/gt/gen6_engine_cs.c
index fe06554729f9..465978ee3a42 100644
--- a/drivers/gpu/drm/i915/gt/gen6_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/gen6_engine_cs.c
@@ -146,7 +146,7 @@ u32 *gen6_emit_breadcrumb_rcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
unsigned int flags;
@@ -187,7 +187,7 @@ u32 *gen6_emit_breadcrumb_rcs(const struct intel_engine_cs *engine,
*cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
@@ -379,7 +379,7 @@ u32 *gen7_emit_breadcrumb_rcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
unsigned int flags;
@@ -407,7 +407,7 @@ u32 *gen7_emit_breadcrumb_rcs(const struct intel_engine_cs *engine,
*cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
@@ -416,7 +416,7 @@ u32 *gen6_emit_breadcrumb_xcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
unsigned int flags = 0;
@@ -436,7 +436,7 @@ u32 *gen6_emit_breadcrumb_xcs(const struct intel_engine_cs *engine,
*cs++ = MI_USER_INTERRUPT;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
@@ -446,7 +446,7 @@ u32 *gen7_emit_breadcrumb_xcs(const struct intel_engine_cs *engien,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
u32 cmd = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW;
int i;
@@ -473,7 +473,7 @@ u32 *gen7_emit_breadcrumb_xcs(const struct intel_engine_cs *engien,
*cs++ = MI_USER_INTERRUPT;
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return cs;
}
diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
index bdc6e5e40b14..2086f10b607b 100644
--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
@@ -351,7 +351,7 @@ int gen8_emit_init_breadcrumb(const struct intel_engine_cs *engine,
u32 *cs;
GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq));
- if (!intel_timeline_has_initial_breadcrumb(i915_request_timeline(rq)))
+ if (!intel_timeline_has_initial_breadcrumb(i915_request_get_timeline(rq)))
return 0;
cs = intel_ring_begin(rq, 6);
@@ -461,10 +461,9 @@ int gen8_emit_bb_start(const struct intel_engine_cs *engine,
static void assert_request_valid(struct i915_request *rq)
{
- struct intel_ring *ring __maybe_unused = rq->ring;
-
/* Can we unwind this request without appearing to go forwards? */
- GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0);
+ GEM_BUG_ON(intel_ring_direction(i915_request_get_ring(rq),
+ rq->wa_tail, rq->head) <= 0);
}
/*
@@ -514,14 +513,14 @@ gen8_emit_fini_breadcrumb_tail(const struct intel_engine_cs *engine,
cs = emit_preempt_busywait(engine, rq, cs);
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return gen8_emit_wa_tail(rq, cs);
}
static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
unsigned int flags = MI_FLUSH_DW_OP_STOREDW;
u32 offset = __i915_request_hwsp_offset(rq);
@@ -548,7 +547,7 @@ u32 *gen8_emit_fini_breadcrumb_rcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
unsigned int flags = PIPE_CONTROL_FLUSH_ENABLE | PIPE_CONTROL_CS_STALL;
u32 offset = __i915_request_hwsp_offset(rq);
@@ -576,7 +575,7 @@ u32 *gen11_emit_fini_breadcrumb_rcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
unsigned int flags;
@@ -648,7 +647,7 @@ gen12_emit_fini_breadcrumb_tail(const struct intel_engine_cs *engine,
cs = gen12_emit_preempt_busywait(engine, rq, cs);
rq->tail = intel_ring_offset(rq, cs);
- assert_ring_tail_valid(rq->ring, rq->tail);
+ assert_ring_tail_valid(i915_request_get_ring(rq), rq->tail);
return gen8_emit_wa_tail(rq, cs);
}
@@ -666,7 +665,7 @@ u32 *gen12_emit_fini_breadcrumb_rcs(const struct intel_engine_cs *engine,
struct i915_request *rq,
u32 *cs)
{
- struct intel_timeline *tl = rcu_dereference_protected(rq->timeline, 1);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
u32 offset = __i915_request_hwsp_offset(rq);
unsigned int flags;
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index c7ab4ed92da4..014b59f18def 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -447,7 +447,7 @@ int intel_context_prepare_remote_request(struct intel_context *ce,
/* Only suitable for use in remotely modifying this context */
GEM_BUG_ON(rq->context == ce);
- if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
+ if (i915_request_get_timeline(rq) != tl) { /* timeline sharing! */
/* Queue this switch after current activity by this context. */
err = i915_active_fence_set(&tl->last_request, rq);
if (err)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index 93b5aa77073f..6e2d7f9d8aa6 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -689,49 +689,27 @@ static int engine_setup_common(struct intel_engine_cs *engine)
return err;
}
-struct measure_breadcrumb {
- struct i915_request rq;
- struct intel_ring ring;
- u32 cs[2048];
-};
-
static int measure_breadcrumb_dw(struct intel_context *ce)
{
struct intel_engine_cs *engine = ce->engine;
- struct i915_sched *se = intel_engine_get_scheduler(engine);
- struct measure_breadcrumb *frame;
+ struct i915_request *rq;
int dw;
GEM_BUG_ON(!engine->gt->scratch);
+ GEM_BUG_ON(ce->ring->emit != 0);
- frame = kzalloc(sizeof(*frame), GFP_KERNEL);
- if (!frame)
+ rq = kzalloc(sizeof(*rq), GFP_KERNEL);
+ if (!rq)
return -ENOMEM;
- frame->rq.context = ce;
- rcu_assign_pointer(frame->rq.timeline, ce->timeline);
- frame->rq.sched.engine = se;
-
- frame->ring.vaddr = frame->cs;
- frame->ring.size = sizeof(frame->cs);
- frame->ring.wrap =
- BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size);
- frame->ring.effective_size = frame->ring.size;
- intel_ring_update_space(&frame->ring);
- frame->rq.ring = &frame->ring;
-
- mutex_lock(&ce->timeline->mutex);
- spin_lock_irq(&se->lock);
-
- dw = engine->emit_fini_breadcrumb(engine, &frame->rq, frame->cs) -
- frame->cs;
-
- spin_unlock_irq(&se->lock);
- mutex_unlock(&ce->timeline->mutex);
+ rq->context = ce;
+ rq->sched.engine = intel_engine_get_scheduler(engine);
+ dw = engine->emit_fini_breadcrumb(engine, rq, ce->ring->vaddr) -
+ (u32 *)ce->ring->vaddr;
GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
- kfree(frame);
+ kfree(rq);
return dw;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index 9ef4e820ba34..8a990a1de057 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -529,7 +529,7 @@ resubmit_virtual_request(struct intel_execlists *el,
if (__i915_request_is_complete(rq))
goto unlock;
- tl = i915_request_active_timeline(rq);
+ tl = i915_request_get_timeline(rq);
/* Rewind back to the start of this virtual engine queue */
list_for_each_entry_continue_reverse(rq, &tl->requests, link) {
@@ -683,10 +683,10 @@ static u64 execlists_update_context(struct i915_request *rq)
* HW has a tendency to ignore us rewinding the TAIL to the end of
* an earlier request.
*/
- GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);
- prev = rq->ring->tail;
- tail = intel_ring_set_tail(rq->ring, rq->tail);
- if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))
+ GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != ce->ring->tail);
+ prev = ce->ring->tail;
+ tail = intel_ring_set_tail(ce->ring, rq->tail);
+ if (unlikely(intel_ring_direction(ce->ring, tail, prev) <= 0))
desc |= CTX_DESC_FORCE_RESTORE;
ce->lrc_reg_state[CTX_RING_TAIL] = tail;
rq->tail = rq->wa_tail;
@@ -3652,11 +3652,11 @@ static int print_ring(char *buf, int sz, struct i915_request *rq)
rcu_read_lock();
if (!i915_request_signaled(rq)) {
- struct intel_timeline *tl = rcu_dereference(rq->timeline);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
len = scnprintf(buf, sz,
"ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ",
- intel_ring_address(rq->ring),
+ intel_ring_address(i915_request_get_ring(rq)),
tl ? tl->ggtt_offset : 0,
hwsp_seqno(rq),
DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context),
diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c
index 521972c297a9..9310cbe23ea7 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring.c
@@ -214,7 +214,7 @@ wait_for_space(struct intel_ring *ring,
GEM_BUG_ON(list_empty(&tl->requests));
list_for_each_entry(target, &tl->requests, link) {
- if (target->ring != ring)
+ if (i915_request_get_ring(target) != ring)
continue;
/* Would completion of this request free enough space? */
@@ -241,7 +241,7 @@ wait_for_space(struct intel_ring *ring,
u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
{
- struct intel_ring *ring = rq->ring;
+ struct intel_ring *ring = i915_request_get_ring(rq);
const unsigned int remain_usable = ring->effective_size - ring->emit;
const unsigned int bytes = num_dwords * sizeof(u32);
unsigned int need_wrap = 0;
@@ -291,7 +291,7 @@ u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
GEM_BUG_ON(!rq->reserved_space);
ret = wait_for_space(ring,
- i915_request_timeline(rq),
+ i915_request_get_timeline(rq),
total_bytes);
if (unlikely(ret))
return ERR_PTR(ret);
@@ -322,10 +322,11 @@ u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
/* Align the ring tail to a cacheline boundary */
int intel_ring_cacheline_align(struct i915_request *rq)
{
+ struct intel_ring *ring = i915_request_get_ring(rq);
int num_dwords;
void *cs;
- num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
+ num_dwords = (ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
if (num_dwords == 0)
return 0;
@@ -339,7 +340,7 @@ int intel_ring_cacheline_align(struct i915_request *rq)
memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
intel_ring_advance(rq, cs + num_dwords);
- GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
+ GEM_BUG_ON(ring->emit & (CACHELINE_BYTES - 1));
return 0;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ring.h b/drivers/gpu/drm/i915/gt/intel_ring.h
index 89d79c22fe9e..aac5ac3a8bc1 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring.h
+++ b/drivers/gpu/drm/i915/gt/intel_ring.h
@@ -50,7 +50,8 @@ static inline void intel_ring_advance(struct i915_request *rq, u32 *cs)
* reserved for the command packet (i.e. the value passed to
* intel_ring_begin()).
*/
- GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs);
+ GEM_BUG_ON((i915_request_get_ring(rq)->vaddr +
+ i915_request_get_ring(rq)->emit) != cs);
}
static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos)
@@ -82,10 +83,11 @@ intel_ring_offset_valid(const struct intel_ring *ring,
static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr)
{
/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
- u32 offset = addr - rq->ring->vaddr;
+ struct intel_ring *ring = i915_request_get_ring(rq);
+ u32 offset = addr - ring->vaddr;
- GEM_BUG_ON(offset > rq->ring->size);
- return intel_ring_wrap(rq->ring, offset);
+ GEM_BUG_ON(offset > ring->size);
+ return intel_ring_wrap(ring, offset);
}
static inline void
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c b/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
index e72233d2a3e4..8df0ce68b10a 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_scheduler.c
@@ -501,7 +501,7 @@ ring_submit(struct intel_engine_cs *engine, struct i915_request *rq)
set_current_context(&engine->legacy.context, rq->context);
}
- ring_copy(ring, rq->ring, rq->head, rq->tail);
+ ring_copy(ring, i915_request_get_ring(rq), rq->head, rq->tail);
return rq;
}
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index 5f47fe30d243..0cc4da36e58f 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -379,7 +379,7 @@ static void reset_rewind(struct intel_engine_cs *engine, bool stalled)
*/
__i915_request_reset(rq, stalled);
- GEM_BUG_ON(rq->ring != engine->legacy.ring);
+ GEM_BUG_ON(i915_request_get_ring(rq) != engine->legacy.ring);
head = rq->head;
} else {
head = engine->legacy.ring->tail;
@@ -891,7 +891,7 @@ ring_context_request(struct intel_context *ce, struct i915_request *rq)
int ret;
GEM_BUG_ON(!intel_context_is_pinned(ce));
- GEM_BUG_ON(intel_timeline_has_initial_breadcrumb(i915_request_timeline(rq)));
+ GEM_BUG_ON(intel_timeline_has_initial_breadcrumb(i915_request_get_timeline(rq)));
/*
* Flush enough space to reduce the likelihood of waiting after
@@ -1144,7 +1144,7 @@ static void __write_tail(struct intel_engine_cs *engine,
struct i915_request *rq)
{
ENGINE_WRITE(engine, RING_TAIL,
- intel_ring_set_tail(rq->ring, rq->tail));
+ intel_ring_set_tail(i915_request_get_ring(rq), rq->tail));
}
static void wa_write_tail(struct intel_engine_cs *engine,
diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c
index a02fd70644e2..8a0388309218 100644
--- a/drivers/gpu/drm/i915/gt/selftest_context.c
+++ b/drivers/gpu/drm/i915/gt/selftest_context.c
@@ -14,7 +14,7 @@
static int request_sync(struct i915_request *rq)
{
- struct intel_timeline *tl = i915_request_timeline(rq);
+ struct intel_timeline *tl = i915_request_get_timeline(rq);
long timeout;
int err = 0;
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
index c4da47f91d72..a86aab1b3354 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_cs.c
@@ -54,7 +54,7 @@ static int write_timestamp(struct i915_request *rq, int slot)
cmd++;
*cs++ = cmd;
*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(engine->mmio_base));
- *cs++ = i915_request_timeline(rq)->ggtt_offset + slot * sizeof(u32);
+ *cs++ = i915_request_get_timeline(rq)->ggtt_offset + slot * sizeof(u32);
*cs++ = 0;
intel_ring_advance(rq, cs);
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 4ac4136be8a0..ee525f60dad7 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2578,8 +2578,8 @@ static int live_chain_preempt(void *arg)
ring_size = rq->wa_tail - rq->head;
if (ring_size < 0)
- ring_size += rq->ring->size;
- ring_size = rq->ring->size / ring_size;
+ ring_size += i915_request_get_ring(rq)->size;
+ ring_size = i915_request_get_ring(rq)->size / ring_size;
pr_debug("%s(%s): Using maximum of %d requests\n",
__func__, engine->name, ring_size);
diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c
index 1089171f5b58..f71034d91d08 100644
--- a/drivers/gpu/drm/i915/gt/selftest_timeline.c
+++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c
@@ -1104,8 +1104,8 @@ static int live_hwsp_read(void *arg)
i915_request_put(rq);
/* Single requests are limited to half a ring at most */
- if (8 * watcher[1].rq->ring->emit >
- 3 * watcher[1].rq->ring->size)
+ if (8 * i915_request_get_ring(watcher[1].rq)->emit >
+ 3 * i915_request_get_ring(watcher[1].rq)->size)
break;
} while (!__igt_timeout(end_time, NULL));
@@ -1172,7 +1172,7 @@ static int live_hwsp_rollover_kernel(void *arg)
engine->name,
lower_32_bits(this->fence.seqno));
- GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);
+ GEM_BUG_ON(i915_request_get_timeline(this) != tl);
rq[i] = i915_request_get(this);
i915_request_add(this);
@@ -1257,7 +1257,7 @@ static int live_hwsp_rollover_user(void *arg)
engine->name,
lower_32_bits(this->fence.seqno));
- GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);
+ GEM_BUG_ON(i915_request_get_timeline(this) != tl);
rq[i] = i915_request_get(this);
i915_request_add(this);
@@ -1453,7 +1453,7 @@ static int live_hwsp_relative(void *arg)
if (IS_ERR(rq))
return PTR_ERR(rq);
- GEM_BUG_ON(rcu_access_pointer(rq->timeline) != tl);
+ GEM_BUG_ON(i915_request_get_timeline(rq) != tl);
i915_request_get(rq);
i915_request_add(rq);
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index bf467a9ca0fc..57cb0025a3cc 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -1020,7 +1020,7 @@ void i915_request_add_active_barriers(struct i915_request *rq)
GEM_BUG_ON(!intel_context_is_barrier(rq->context));
GEM_BUG_ON(intel_engine_is_virtual(engine));
- GEM_BUG_ON(i915_request_timeline(rq) != engine->kernel_context->timeline);
+ GEM_BUG_ON(i915_request_get_timeline(rq) != engine->kernel_context->timeline);
node = llist_del_all(&engine->barrier_tasks);
if (!node)
diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h
index fb165d3f01cf..2ab503e1ebcc 100644
--- a/drivers/gpu/drm/i915/i915_active.h
+++ b/drivers/gpu/drm/i915/i915_active.h
@@ -170,9 +170,7 @@ int i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence);
static inline int
i915_active_add_request(struct i915_active *ref, struct i915_request *rq)
{
- return i915_active_ref(ref,
- i915_request_timeline(rq)->fence_context,
- &rq->fence);
+ return i915_active_ref(ref, rq->fence.context, &rq->fence);
}
struct dma_fence *
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index ed20fb2181c2..7430f68d2ced 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -1320,7 +1320,7 @@ intel_engine_coredump_add_request(struct intel_engine_coredump *ee,
*/
vma = capture_vma(vma, rq->batch, "batch", gfp);
vma = capture_user(vma, rq, gfp);
- vma = capture_vma(vma, rq->ring->vma, "ring", gfp);
+ vma = capture_vma(vma, i915_request_get_ring(rq)->vma, "ring", gfp);
vma = capture_vma(vma, rq->context->state, "HW context", gfp);
ee->rq_head = rq->head;
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 21ca09884447..4203f41980dc 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -223,12 +223,12 @@ static void free_capture_list(struct i915_request *request)
static void __i915_request_fill(struct i915_request *rq, u8 val)
{
- void *vaddr = rq->ring->vaddr;
+ void *vaddr = i915_request_get_ring(rq)->vaddr;
u32 head;
head = rq->infix;
if (rq->postfix < head) {
- memset(vaddr + head, val, rq->ring->size - head);
+ memset(vaddr + head, val, i915_request_get_ring(rq)->size - head);
head = 0;
}
memset(vaddr + head, val, rq->postfix - head);
@@ -285,11 +285,11 @@ bool i915_request_retire(struct i915_request *rq)
* completion order.
*/
GEM_BUG_ON(!list_is_first(&rq->link,
- &i915_request_timeline(rq)->requests));
+ &i915_request_get_timeline(rq)->requests));
if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
/* Poison before we release our space in the ring */
__i915_request_fill(rq, POISON_FREE);
- rq->ring->head = rq->postfix;
+ i915_request_get_ring(rq)->head = rq->postfix;
if (!i915_request_signaled(rq)) {
spin_lock_irq(&rq->lock);
@@ -328,7 +328,7 @@ bool i915_request_retire(struct i915_request *rq)
void i915_request_retire_upto(struct i915_request *rq)
{
- struct intel_timeline * const tl = i915_request_timeline(rq);
+ struct intel_timeline * const tl = i915_request_get_timeline(rq);
struct i915_request *tmp;
RQ_TRACE(rq, "\n");
@@ -509,7 +509,8 @@ bool __i915_request_submit(struct i915_request *request,
request->context->saturated |= request->sched.semaphores;
engine->emit_fini_breadcrumb(engine, request,
- request->ring->vaddr + request->postfix);
+ i915_request_get_ring(request)->vaddr +
+ request->postfix);
trace_i915_request_execute(request);
engine->serial++;
@@ -671,6 +672,7 @@ struct i915_request *
__i915_request_create(struct intel_context *ce, gfp_t gfp)
{
struct intel_timeline *tl = ce->timeline;
+ struct intel_ring *ring = ce->ring;
struct i915_request *rq;
u32 seqno;
int ret;
@@ -720,7 +722,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
}
rq->context = ce;
- rq->ring = ce->ring;
kref_init(&rq->fence.refcount);
rq->fence.flags = 0;
@@ -734,7 +735,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
rq->fence.context = tl->fence_context;
rq->fence.seqno = seqno;
- RCU_INIT_POINTER(rq->timeline, tl);
RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline);
rq->hwsp_seqno = tl->hwsp_seqno;
GEM_BUG_ON(__i915_request_is_complete(rq));
@@ -769,13 +769,13 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
* GPU processing the request, we never over-estimate the
* position of the head.
*/
- rq->head = rq->ring->emit;
+ rq->head = ring->emit;
ret = ce->ops->init_request(ce, rq);
if (ret)
goto err_unwind;
- rq->infix = rq->ring->emit; /* end of header; start of user payload */
+ rq->infix = ring->emit; /* end of header; start of user payload */
intel_context_mark_active(ce);
list_add_tail_rcu(&rq->link, &tl->requests);
@@ -783,7 +783,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp)
return rq;
err_unwind:
- ce->ring->emit = rq->head;
+ ring->emit = rq->head;
/* Make sure we didn't add ourselves to external state before freeing */
GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
@@ -833,7 +833,7 @@ i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
struct dma_fence *fence;
int err;
- if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline))
+ if (i915_request_get_timeline(rq) == i915_request_get_timeline(signal))
return 0;
if (i915_request_started(signal))
@@ -858,7 +858,7 @@ i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
break;
/* Is signal the earliest request on its timeline? */
- if (pos == &rcu_dereference(signal->timeline)->requests)
+ if (pos == &i915_request_get_timeline(signal)->requests)
break;
/*
@@ -884,7 +884,7 @@ i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
return 0;
err = 0;
- if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))
+ if (!intel_timeline_sync_is_later(i915_request_get_timeline(rq), fence))
err = i915_sw_fence_await_dma_fence(&rq->sched.submit,
fence, 0,
I915_FENCE_GFP);
@@ -1041,7 +1041,7 @@ __i915_request_await_execution(struct i915_request *to,
return err;
/* Squash repeated depenendices to the same timelines */
- if (intel_timeline_sync_has_start(i915_request_timeline(to),
+ if (intel_timeline_sync_has_start(i915_request_get_timeline(to),
&from->fence))
return 0;
@@ -1094,7 +1094,7 @@ __i915_request_await_execution(struct i915_request *to,
return err;
}
- return intel_timeline_sync_set_start(i915_request_timeline(to),
+ return intel_timeline_sync_set_start(i915_request_get_timeline(to),
&from->fence);
}
@@ -1221,8 +1221,8 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from)
int ret;
GEM_BUG_ON(to == from);
- GEM_BUG_ON(to->timeline == from->timeline);
GEM_BUG_ON(to->context == from->context);
+ GEM_BUG_ON(to->context->timeline == from->context->timeline);
if (i915_request_completed(from)) {
i915_sw_fence_set_error_once(&to->sched.submit,
@@ -1289,7 +1289,7 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
/* Squash repeated waits to the same timelines */
if (fence->context &&
- intel_timeline_sync_is_later(i915_request_timeline(rq),
+ intel_timeline_sync_is_later(i915_request_get_timeline(rq),
fence))
continue;
@@ -1302,7 +1302,7 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
/* Record the latest fence used against each timeline */
if (fence->context)
- intel_timeline_sync_set(i915_request_timeline(rq),
+ intel_timeline_sync_set(i915_request_get_timeline(rq),
fence);
} while (--nchild);
@@ -1383,7 +1383,7 @@ static bool in_order_submission(const struct i915_request *prev,
static struct i915_request *
__i915_request_add_to_timeline(struct i915_request *rq)
{
- struct intel_timeline *timeline = i915_request_timeline(rq);
+ struct intel_timeline *timeline = i915_request_get_timeline(rq);
struct i915_request *prev;
/*
@@ -1452,7 +1452,7 @@ __i915_request_add_to_timeline(struct i915_request *rq)
struct i915_request *__i915_request_commit(struct i915_request *rq)
{
struct intel_engine_cs *engine = i915_request_get_engine(rq);
- struct intel_ring *ring = rq->ring;
+ struct intel_ring *ring = i915_request_get_ring(rq);
u32 *cs;
RQ_TRACE(rq, "\n");
@@ -1509,7 +1509,7 @@ void __i915_request_queue(struct i915_request *rq,
void i915_request_add(struct i915_request *rq)
{
- struct intel_timeline * const tl = i915_request_timeline(rq);
+ struct intel_timeline * const tl = i915_request_get_timeline(rq);
struct i915_sched_attr attr = {};
struct i915_gem_context *ctx;
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index dfd74b32f10d..babe7ff8a181 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -177,9 +177,6 @@ struct i915_request {
* context.
*/
struct intel_context *context;
- struct intel_ring *ring;
- struct intel_timeline __rcu *timeline;
-
struct list_head signal_link;
struct llist_node signal_node;
@@ -603,11 +600,15 @@ i915_request_get_engine(const struct i915_request *rq)
}
static inline struct intel_timeline *
-i915_request_timeline(const struct i915_request *rq)
+i915_request_get_timeline(const struct i915_request *rq)
{
- /* Valid only while the request is being constructed (or retired). */
- return rcu_dereference_protected(rq->timeline,
- lockdep_is_held(&rcu_access_pointer(rq->timeline)->mutex));
+ return rq->context->timeline;
+}
+
+static inline struct intel_ring *
+i915_request_get_ring(const struct i915_request *rq)
+{
+ return rq->context->ring;
}
static inline struct i915_gem_context *
@@ -617,18 +618,6 @@ i915_request_gem_context(const struct i915_request *rq)
return rcu_dereference_protected(rq->context->gem_context, true);
}
-static inline struct intel_timeline *
-i915_request_active_timeline(const struct i915_request *rq)
-{
- /*
- * When in use during submission, we are protected by a guarantee that
- * the context/timeline is pinned and must remain pinned until after
- * this submission.
- */
- return rcu_dereference_protected(rq->timeline,
- lockdep_is_held(&i915_request_get_scheduler(rq)->lock));
-}
-
static inline u32 __i915_request_hwsp_offset(const struct i915_request *rq)
{
const struct intel_timeline_cacheline *cl;
@@ -639,7 +628,7 @@ static inline u32 __i915_request_hwsp_offset(const struct i915_request *rq)
if (cl)
return cl->ggtt_offset;
- return rcu_dereference_protected(rq->timeline, 1)->ggtt_offset;
+ return i915_request_get_timeline(rq)->ggtt_offset;
}
static inline bool i915_request_use_scheduler(const struct i915_request *rq)
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index c0b4fa561662..d42ac6ada2de 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -134,7 +134,7 @@ static void i915_sched_init_ipi(struct i915_sched_ipi *ipi)
static bool match_ring(const struct i915_request *rq)
{
const struct intel_engine_cs *engine = i915_request_get_engine(rq);
- const struct intel_ring *ring = rq->ring;
+ const struct intel_ring *ring = i915_request_get_ring(rq);
if (intel_ring_is_internal(ring))
return true;
@@ -791,7 +791,7 @@ static u64 signal_deadline(const struct i915_request *rq)
if (__i915_request_is_complete(s))
continue;
- if (s->timeline == rq->timeline &&
+ if (i915_request_get_timeline(s) == i915_request_get_timeline(rq) &&
__i915_request_has_started(s))
continue;
@@ -1257,9 +1257,9 @@ struct i915_request *__i915_sched_rewind_requests(struct i915_sched *se)
set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
/* Check in case we rollback so far we wrap [size/2] */
- if (intel_ring_direction(rq->ring,
+ if (intel_ring_direction(i915_request_get_ring(rq),
rq->tail,
- rq->ring->tail + 8) > 0)
+ i915_request_get_ring(rq)->tail + 8) > 0)
rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;
active = rq;
@@ -1705,7 +1705,7 @@ void i915_request_show_with_schedule(struct drm_printer *m,
node_to_request(dep->signaler);
/* Dependencies along the same timeline are expected. */
- if (signaler->timeline == rq->timeline)
+ if (i915_request_get_timeline(signaler) == i915_request_get_timeline(rq))
continue;
if (__i915_request_is_complete(signaler))
@@ -1770,16 +1770,16 @@ print_request_ring(struct drm_printer *m, const struct i915_request *rq)
size = rq->tail - rq->head;
if (rq->tail < rq->head)
- size += rq->ring->size;
+ size += i915_request_get_ring(rq)->size;
ring = kmalloc(size, GFP_ATOMIC);
if (ring) {
- const void *vaddr = rq->ring->vaddr;
+ const void *vaddr = i915_request_get_ring(rq)->vaddr;
unsigned int head = rq->head;
unsigned int len = 0;
if (rq->tail < head) {
- len = rq->ring->size - head;
+ len = i915_request_get_ring(rq)->size - head;
memcpy(ring, vaddr + head, len);
head = 0;
}
@@ -1822,20 +1822,18 @@ void i915_sched_show(struct drm_printer *m,
rq = se->active_request(se);
if (rq) {
+ struct intel_ring *ring = i915_request_get_ring(rq);
+
i915_request_show(m, rq, "\t\tactive ", 0);
drm_printf(m, "\t\tring->start: 0x%08x\n",
- intel_ring_address(rq->ring));
- drm_printf(m, "\t\tring->head: 0x%08x\n",
- rq->ring->head);
- drm_printf(m, "\t\tring->tail: 0x%08x\n",
- rq->ring->tail);
- drm_printf(m, "\t\tring->emit: 0x%08x\n",
- rq->ring->emit);
- drm_printf(m, "\t\tring->space: 0x%08x\n",
- rq->ring->space);
+ intel_ring_address(ring));
+ drm_printf(m, "\t\tring->head: 0x%08x\n", ring->head);
+ drm_printf(m, "\t\tring->tail: 0x%08x\n", ring->tail);
+ drm_printf(m, "\t\tring->emit: 0x%08x\n", ring->emit);
+ drm_printf(m, "\t\tring->space: 0x%08x\n", ring->space);
drm_printf(m, "\t\tring->hwsp: 0x%08x\n",
- i915_request_active_timeline(rq)->ggtt_offset);
+ i915_request_get_timeline(rq)->ggtt_offset);
print_request_ring(m, rq);
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 8059888f8eef..f7f7808eedea 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -1255,14 +1255,15 @@ max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
if (IS_ERR(rq)) {
ret = PTR_ERR(rq);
} else {
+ struct intel_ring *ring = i915_request_get_ring(rq);
int sz;
- ret = rq->ring->size - rq->reserved_space;
+ ret = ring->size - rq->reserved_space;
i915_request_add(rq);
- sz = rq->ring->emit - rq->head;
+ sz = ring->emit - rq->head;
if (sz < 0)
- sz += rq->ring->size;
+ sz += ring->size;
ret /= sz;
ret /= 2; /* leave half spare, in case of emergency! */
}
--
2.20.1
More information about the Intel-gfx-trybot
mailing list