[Intel-gfx] [igt-dev] [PATCH i-g-t 17/21] gem_wsim: Infinite batch support
Chris Wilson
chris at chris-wilson.co.uk
Fri May 10 13:48:51 UTC 2019
Quoting Tvrtko Ursulin (2019-05-08 13:10:54)
> From: Tvrtko Ursulin <tvrtko.ursulin at intel.com>
>
> For simulating frame split workloads it is useful to express a batch which
> ends at the same time as the parallel submission on the respective bonded
> engine. For this we add support for infinite batch durations and the batch
> terminate command ('T'). Syntax looks like this:
>
> 1.RCS.*.0.0
> T.-1
>
> First step starts an infinite batch, and second command terminates the
> infinite batch with the usual relative workload step addressing.
>
> Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin at intel.com>
> ---
> benchmarks/gem_wsim.c | 119 +++++++++++++++++++------
> benchmarks/wsim/README | 9 +-
> benchmarks/wsim/frame-split-60fps.wsim | 6 +-
> 3 files changed, 102 insertions(+), 32 deletions(-)
>
> diff --git a/benchmarks/gem_wsim.c b/benchmarks/gem_wsim.c
> index cc6f4a742c12..97821b723b02 100644
> --- a/benchmarks/gem_wsim.c
> +++ b/benchmarks/gem_wsim.c
> @@ -86,6 +86,7 @@ enum w_type
> ENGINE_MAP,
> LOAD_BALANCE,
> BOND,
> + TERMINATE,
> };
>
> struct deps
> @@ -113,6 +114,7 @@ struct w_step
> unsigned int context;
> unsigned int engine;
> struct duration duration;
> + bool unbound_duration;
> struct deps data_deps;
> struct deps fence_deps;
> int emit_fence;
> @@ -143,7 +145,7 @@ struct w_step
>
> struct drm_i915_gem_execbuffer2 eb;
> struct drm_i915_gem_exec_object2 *obj;
> - struct drm_i915_gem_relocation_entry reloc[4];
> + struct drm_i915_gem_relocation_entry reloc[5];
> unsigned long bb_sz;
> uint32_t bb_handle;
> uint32_t *seqno_value;
> @@ -153,6 +155,7 @@ struct w_step
> uint32_t *rt1_address;
> uint32_t *latch_value;
> uint32_t *latch_address;
> + uint32_t *recursive_bb_start;
> };
>
> DECLARE_EWMA(uint64_t, rt, 4, 2)
> @@ -491,6 +494,10 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
>
> step.type = ENGINE_MAP;
> goto add_step;
> + } else if (!strcmp(field, "T")) {
> + int_field(TERMINATE, target,
> + tmp >= 0 || ((int)nr_steps + tmp) < 0,
> + "Invalid terminate target at step %u!\n");
> } else if (!strcmp(field, "X")) {
> unsigned int nr = 0;
> while ((field = strtok_r(fstart, ".", &fctx))) {
> @@ -605,23 +612,28 @@ parse_workload(struct w_arg *arg, unsigned int flags, struct workload *app_w)
>
> fstart = NULL;
>
> - tmpl = strtol(field, &sep, 10);
> - check_arg(tmpl <= 0 || tmpl == LONG_MIN ||
> - tmpl == LONG_MAX,
> - "Invalid duration at step %u!\n", nr_steps);
> - step.duration.min = tmpl;
> -
> - if (sep && *sep == '-') {
> - tmpl = strtol(sep + 1, NULL, 10);
> - check_arg(tmpl <= 0 ||
> - tmpl <= step.duration.min ||
> - tmpl == LONG_MIN ||
> + if (field[0] == '*') {
> + step.unbound_duration = true;
> + } else {
> + tmpl = strtol(field, &sep, 10);
> + check_arg(tmpl <= 0 || tmpl == LONG_MIN ||
> tmpl == LONG_MAX,
> - "Invalid duration range at step %u!\n",
> + "Invalid duration at step %u!\n",
> nr_steps);
> - step.duration.max = tmpl;
> - } else {
> - step.duration.max = step.duration.min;
> + step.duration.min = tmpl;
> +
> + if (sep && *sep == '-') {
> + tmpl = strtol(sep + 1, NULL, 10);
> + check_arg(tmpl <= 0 ||
> + tmpl <= step.duration.min ||
> + tmpl == LONG_MIN ||
> + tmpl == LONG_MAX,
> + "Invalid duration range at step %u!\n",
> + nr_steps);
> + step.duration.max = tmpl;
> + } else {
> + step.duration.max = step.duration.min;
> + }
> }
>
> valid++;
> @@ -781,7 +793,7 @@ init_bb(struct w_step *w, unsigned int flags)
> unsigned int i;
> uint32_t *ptr;
>
> - if (!arb_period)
> + if (w->unbound_duration || !arb_period)
> return;
>
> gem_set_domain(fd, w->bb_handle,
> @@ -801,6 +813,7 @@ terminate_bb(struct w_step *w, unsigned int flags)
> const uint32_t bbe = 0xa << 23;
> unsigned long mmap_start, mmap_len;
> unsigned long batch_start = w->bb_sz;
> + unsigned int r = 0;
> uint32_t *ptr, *cs;
>
> igt_assert(((flags & RT) && (flags & SEQNO)) || !(flags & RT));
> @@ -811,6 +824,9 @@ terminate_bb(struct w_step *w, unsigned int flags)
> if (flags & RT)
> batch_start -= 12 * sizeof(uint32_t);
>
> + if (w->unbound_duration)
> + batch_start -= 4 * sizeof(uint32_t); /* MI_ARB_CHK + MI_BATCH_BUFFER_START */
> +
> mmap_start = rounddown(batch_start, PAGE_SIZE);
> mmap_len = ALIGN(w->bb_sz - mmap_start, PAGE_SIZE);
>
> @@ -820,8 +836,19 @@ terminate_bb(struct w_step *w, unsigned int flags)
> ptr = gem_mmap__wc(fd, w->bb_handle, mmap_start, mmap_len, PROT_WRITE);
> cs = (uint32_t *)((char *)ptr + batch_start - mmap_start);
>
> + if (w->unbound_duration) {
> + w->reloc[r++].offset = batch_start + 2 * sizeof(uint32_t);
> + batch_start += 4 * sizeof(uint32_t);
> +
> + *cs++ = w->preempt_us ? 0x5 << 23 /* MI_ARB_CHK; */ : MI_NOOP;
> + w->recursive_bb_start = cs;
> + *cs++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
> + *cs++ = 0;
> + *cs++ = 0;
Hmm. Have we previously checked for gen >= 8?
So preemption check interval is given by batch_start - mmap_start.
Which is limited to a max of 64 bytes. That might be a bit excessive on
the frequency of doing MI_BB_START, certainly for gen7, gen8+ is a tad
more forgiving i.e. it has more bw and doesn't starve the cpu as much.
> + }
> +
> if (flags & SEQNO) {
> - w->reloc[0].offset = batch_start + sizeof(uint32_t);
> + w->reloc[r++].offset = batch_start + sizeof(uint32_t);
> batch_start += 4 * sizeof(uint32_t);
>
> *cs++ = MI_STORE_DWORD_IMM;
> @@ -833,7 +860,7 @@ terminate_bb(struct w_step *w, unsigned int flags)
> }
>
> if (flags & RT) {
> - w->reloc[1].offset = batch_start + sizeof(uint32_t);
> + w->reloc[r++].offset = batch_start + sizeof(uint32_t);
> batch_start += 4 * sizeof(uint32_t);
>
> *cs++ = MI_STORE_DWORD_IMM;
> @@ -843,7 +870,7 @@ terminate_bb(struct w_step *w, unsigned int flags)
> w->rt0_value = cs;
> *cs++ = 0;
>
> - w->reloc[2].offset = batch_start + 2 * sizeof(uint32_t);
> + w->reloc[r++].offset = batch_start + 2 * sizeof(uint32_t);
> batch_start += 4 * sizeof(uint32_t);
>
> *cs++ = 0x24 << 23 | 2; /* MI_STORE_REG_MEM */
> @@ -852,7 +879,7 @@ terminate_bb(struct w_step *w, unsigned int flags)
> *cs++ = 0;
> *cs++ = 0;
>
> - w->reloc[3].offset = batch_start + sizeof(uint32_t);
> + w->reloc[r++].offset = batch_start + sizeof(uint32_t);
> batch_start += 4 * sizeof(uint32_t);
>
> *cs++ = MI_STORE_DWORD_IMM;
> @@ -984,19 +1011,28 @@ alloc_step_batch(struct workload *wrk, struct w_step *w, unsigned int flags)
> }
> }
>
> - w->bb_sz = get_bb_sz(w->duration.max);
> - w->bb_handle = w->obj[j].handle = gem_create(fd, w->bb_sz);
> + if (w->unbound_duration)
> + /* nops + MI_ARB_CHK + MI_BATCH_BUFFER_START */
> + w->bb_sz = max(64, get_bb_sz(w->preempt_us)) +
> + (1 + 3) * sizeof(uint32_t);
> + else
> + w->bb_sz = get_bb_sz(w->duration.max);
> + w->bb_handle = w->obj[j].handle = gem_create(fd, w->bb_sz + (w->unbound_duration ? 4096 : 0));
> init_bb(w, flags);
> terminate_bb(w, flags);
>
> - if (flags & SEQNO) {
> + if ((flags & SEQNO) || w->unbound_duration) {
> w->obj[j].relocs_ptr = to_user_pointer(&w->reloc);
> + if (flags & SEQNO)
> + w->obj[j].relocation_count++;
> if (flags & RT)
> - w->obj[j].relocation_count = 4;
> - else
> - w->obj[j].relocation_count = 1;
> + w->obj[j].relocation_count += 3;
> + if (w->unbound_duration)
> + w->obj[j].relocation_count++;
Huh, I expected to see w->obj[j].relocation_count = r;
Already out of scope?
> for (i = 0; i < w->obj[j].relocation_count; i++)
> w->reloc[i].target_handle = 1;
> + if (w->unbound_duration)
> + w->reloc[0].target_handle = j;
Ok, recursive BB_START.
> }
>
> w->eb.buffers_ptr = to_user_pointer(w->obj);
> @@ -2036,6 +2072,18 @@ update_bb_rt(struct w_step *w, enum intel_engine_id engine, uint32_t seqno)
> }
> }
>
> +static void
> +update_bb_start(struct w_step *w)
> +{
> + if (!w->unbound_duration)
> + return;
> +
> + gem_set_domain(fd, w->bb_handle,
> + I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC);
Hmm. A scary sync point. Do you just want to be sure you have flushed
the previous user?
> + *w->recursive_bb_start = MI_BATCH_BUFFER_START | (1 << 8) | 1;
> +}
> +
> static void w_sync_to(struct workload *wrk, struct w_step *w, int target)
> {
> if (target < 0)
> @@ -2171,9 +2219,13 @@ do_eb(struct workload *wrk, struct w_step *w, enum intel_engine_id engine,
> if (flags & RT)
> update_bb_rt(w, engine, seqno);
>
> + update_bb_start(w);
> +
> w->eb.batch_start_offset =
> + w->unbound_duration ?
> + 0 :
> ALIGN(w->bb_sz - get_bb_sz(get_duration(w)),
> - 2 * sizeof(uint32_t));
> + 2 * sizeof(uint32_t));
>
> for (i = 0; i < w->fence_deps.nr; i++) {
> int tgt = w->idx + w->fence_deps.list[i];
> @@ -2313,6 +2365,17 @@ static void *run_workload(void *data)
> w->priority;
> }
> continue;
> + } else if (w->type == TERMINATE) {
> + unsigned int t_idx = i + w->target;
> +
> + igt_assert(t_idx >= 0 && t_idx < i);
> + igt_assert(wrk->steps[t_idx].type == BATCH);
> + igt_assert(wrk->steps[t_idx].unbound_duration);
> +
> + *wrk->steps[t_idx].recursive_bb_start =
> + MI_BATCH_BUFFER_END;
> + __sync_synchronize();
> + continue;
> } else if (w->type == PREEMPTION ||
> w->type == ENGINE_MAP ||
> w->type == LOAD_BALANCE ||
> diff --git a/benchmarks/wsim/README b/benchmarks/wsim/README
> index 6aec718bc812..c94d01018419 100644
> --- a/benchmarks/wsim/README
> +++ b/benchmarks/wsim/README
> @@ -2,11 +2,11 @@ Workload descriptor format
> ==========================
>
> ctx.engine.duration_us.dependency.wait,...
> -<uint>.<str>.<uint>[-<uint>].<int <= 0>[/<int <= 0>][...].<0|1>,...
> +<uint>.<str>.<uint>[-<uint>]|*.<int <= 0>[/<int <= 0>][...].<0|1>,...
> B.<uint>
> M.<uint>.<str>[|<str>]...
> P|X.<uint>.<int>
> -d|p|s|t|q|a.<int>,...
> +d|p|s|t|q|a|T.<int>,...
> b.<uint>.<uint>.<str>
> f
>
> @@ -30,6 +30,7 @@ Additional workload steps are also supported:
> 'b' - Set up engine bonds.
> 'M' - Set up engine map.
> 'P' - Context priority.
> + 'T' - Terminate an infinite batch.
> 'X' - Context preemption control.
>
> Engine ids: DEFAULT, RCS, BCS, VCS, VCS1, VCS2, VECS
> @@ -77,6 +78,10 @@ Example:
>
> I this case the last step has a data dependency on both first and second steps.
>
> +Batch durations can also be specified as infinite by using the '*' in the
> +duration field. Such batches must be ended by the terminate command ('T')
> +otherwise they will cause a GPU hang to be reported.
> +
> Sync (fd) fences
> ----------------
>
> diff --git a/benchmarks/wsim/frame-split-60fps.wsim b/benchmarks/wsim/frame-split-60fps.wsim
> index cfbfcd39be7d..ea89da3add48 100644
> --- a/benchmarks/wsim/frame-split-60fps.wsim
> +++ b/benchmarks/wsim/frame-split-60fps.wsim
> @@ -6,10 +6,12 @@ M.2.VCS2
> B.2
> b.2.1.VCS1
> f
> -1.DEFAULT.4000-6000.f-1.0
> +1.DEFAULT.*.f-1.0
> 2.DEFAULT.4000-6000.s-1.0
> a.-3
> -3.RCS.2000-4000.-3/-2.0
> +s.-2
> +T.-4
> +3.RCS.2000-4000.-5/-4.0
> 3.VECS.2000.-1.0
> 4.BCS.1000.-1.0
> s.-2
Usecase looks reasonable.
Reviewed-by: Chris Wilson <chris at chris-wilson.co.uk>
-Chris
More information about the Intel-gfx
mailing list