Mesa (main): ir3/postsched: Rewrite delay handling
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Wed Nov 17 14:16:15 UTC 2021
Module: Mesa
Branch: main
Commit: 6f5c0d209c3949ff2522afa9965fa69811adcb9b
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=6f5c0d209c3949ff2522afa9965fa69811adcb9b
Author: Connor Abbott <cwabbott0 at gmail.com>
Date: Wed Nov 3 18:06:17 2021 +0100
ir3/postsched: Rewrite delay handling
Analogous to the pre-RA scheduler. Unfortunately this time it's a bit
more involved because we have to correctly handle (rptN), which is
already relevant for swz. This means we need the index of the
destination register that conflicts with the source register, to handle
swz, and we need to expose that part of ir3_delay. But once that's done,
we can delete ir3_delay_calc_postra.
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/13722>
---
src/freedreno/ir3/ir3.h | 10 ++--
src/freedreno/ir3/ir3_delay.c | 110 ++++++++++++++++++--------------------
src/freedreno/ir3/ir3_legalize.c | 2 +-
src/freedreno/ir3/ir3_postsched.c | 84 +++++++++++++++++++----------
src/freedreno/ir3/tests/delay.c | 2 +-
5 files changed, 114 insertions(+), 94 deletions(-)
diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h
index 6e4e81270a2..bfbfb3ba9f1 100644
--- a/src/freedreno/ir3/ir3.h
+++ b/src/freedreno/ir3/ir3.h
@@ -1632,11 +1632,11 @@ void ir3_print_instr_stream(struct log_stream *stream, struct ir3_instruction *i
/* delay calculation: */
int ir3_delayslots(struct ir3_instruction *assigner,
struct ir3_instruction *consumer, unsigned n, bool soft);
-unsigned ir3_delay_calc_postra(struct ir3_block *block,
- struct ir3_instruction *instr, bool soft,
- bool mergedregs);
-unsigned ir3_delay_calc_exact(struct ir3_block *block,
- struct ir3_instruction *instr, bool mergedregs);
+unsigned ir3_delayslots_with_repeat(struct ir3_instruction *assigner,
+ struct ir3_instruction *consumer,
+ unsigned assigner_n, unsigned consumer_n);
+unsigned ir3_delay_calc(struct ir3_block *block,
+ struct ir3_instruction *instr, bool mergedregs);
void ir3_remove_nops(struct ir3 *ir);
/* unreachable block elimination: */
diff --git a/src/freedreno/ir3/ir3_delay.c b/src/freedreno/ir3/ir3_delay.c
index f4a748cc3eb..d1920b0b4c1 100644
--- a/src/freedreno/ir3/ir3_delay.c
+++ b/src/freedreno/ir3/ir3_delay.c
@@ -139,35 +139,21 @@ post_ra_reg_num(struct ir3_register *reg)
return reg->num;
}
-static unsigned
-delay_calc_srcn_postra(struct ir3_instruction *assigner,
- struct ir3_instruction *consumer, unsigned assigner_n,
- unsigned consumer_n, bool soft, bool mergedregs)
+unsigned
+ir3_delayslots_with_repeat(struct ir3_instruction *assigner,
+ struct ir3_instruction *consumer,
+ unsigned assigner_n, unsigned consumer_n)
{
+ unsigned delay = ir3_delayslots(assigner, consumer, consumer_n, false);
+
struct ir3_register *src = consumer->srcs[consumer_n];
struct ir3_register *dst = assigner->dsts[assigner_n];
- bool mismatched_half =
- (src->flags & IR3_REG_HALF) != (dst->flags & IR3_REG_HALF);
- /* In the mergedregs case or when the register is a special register,
- * half-registers do not alias with full registers.
- */
- if ((!mergedregs || is_reg_special(src) || is_reg_special(dst)) &&
- mismatched_half)
- return 0;
+ if (assigner->repeat == 0 && consumer->repeat == 0)
+ return delay;
unsigned src_start = post_ra_reg_num(src) * reg_elem_size(src);
- unsigned src_end = src_start + post_ra_reg_elems(src) * reg_elem_size(src);
unsigned dst_start = post_ra_reg_num(dst) * reg_elem_size(dst);
- unsigned dst_end = dst_start + post_ra_reg_elems(dst) * reg_elem_size(dst);
-
- if (dst_start >= src_end || src_start >= dst_end)
- return 0;
-
- unsigned delay = ir3_delayslots(assigner, consumer, consumer_n, soft);
-
- if (assigner->repeat == 0 && consumer->repeat == 0)
- return delay;
/* If either side is a relative access, we can't really apply most of the
* reasoning below because we don't know which component aliases which.
@@ -182,6 +168,9 @@ delay_calc_srcn_postra(struct ir3_instruction *assigner,
if (assigner->opc == OPC_MOVMSK)
return delay;
+ bool mismatched_half =
+ (src->flags & IR3_REG_HALF) != (dst->flags & IR3_REG_HALF);
+
/* TODO: Handle the combination of (rpt) and different component sizes
* better like below. This complicates things significantly because the
* components don't line up.
@@ -235,9 +224,37 @@ delay_calc_srcn_postra(struct ir3_instruction *assigner,
}
static unsigned
-delay_calc_postra(struct ir3_block *block, struct ir3_instruction *start,
- struct ir3_instruction *consumer, unsigned distance,
- bool soft, bool pred, regmask_t *in_mask, bool mergedregs)
+delay_calc_srcn(struct ir3_instruction *assigner,
+ struct ir3_instruction *consumer, unsigned assigner_n,
+ unsigned consumer_n, bool mergedregs)
+{
+ struct ir3_register *src = consumer->srcs[consumer_n];
+ struct ir3_register *dst = assigner->dsts[assigner_n];
+ bool mismatched_half =
+ (src->flags & IR3_REG_HALF) != (dst->flags & IR3_REG_HALF);
+
+ /* In the mergedregs case or when the register is a special register,
+ * half-registers do not alias with full registers.
+ */
+ if ((!mergedregs || is_reg_special(src) || is_reg_special(dst)) &&
+ mismatched_half)
+ return 0;
+
+ unsigned src_start = post_ra_reg_num(src) * reg_elem_size(src);
+ unsigned src_end = src_start + post_ra_reg_elems(src) * reg_elem_size(src);
+ unsigned dst_start = post_ra_reg_num(dst) * reg_elem_size(dst);
+ unsigned dst_end = dst_start + post_ra_reg_elems(dst) * reg_elem_size(dst);
+
+ if (dst_start >= src_end || src_start >= dst_end)
+ return 0;
+
+ return ir3_delayslots_with_repeat(assigner, consumer, assigner_n, consumer_n);
+}
+
+static unsigned
+delay_calc(struct ir3_block *block, struct ir3_instruction *start,
+ struct ir3_instruction *consumer, unsigned distance,
+ regmask_t *in_mask, bool mergedregs)
{
regmask_t mask;
memcpy(&mask, in_mask, sizeof(mask));
@@ -253,7 +270,7 @@ delay_calc_postra(struct ir3_block *block, struct ir3_instruction *start,
if (count_instruction(assigner))
distance += assigner->nop;
- if (distance + delay >= (soft ? SOFT_SS_NOPS : MAX_NOPS))
+ if (distance + delay >= MAX_NOPS)
return delay;
if (is_meta(assigner))
@@ -270,8 +287,8 @@ delay_calc_postra(struct ir3_block *block, struct ir3_instruction *start,
if (src->flags & (IR3_REG_IMMED | IR3_REG_CONST))
continue;
- unsigned src_delay = delay_calc_srcn_postra(
- assigner, consumer, dst_n, src_n, soft, mergedregs);
+ unsigned src_delay = delay_calc_srcn(
+ assigner, consumer, dst_n, src_n, mergedregs);
new_delay = MAX2(new_delay, src_delay);
}
regmask_clear(&mask, dst);
@@ -298,13 +315,13 @@ delay_calc_postra(struct ir3_block *block, struct ir3_instruction *start,
* However any other recursion would be unnecessary.
*/
- if (pred && block->data != block) {
+ if (block->data != block) {
block->data = block;
for (unsigned i = 0; i < block->predecessors_count; i++) {
struct ir3_block *pred = block->predecessors[i];
- unsigned pred_delay = delay_calc_postra(pred, NULL, consumer, distance,
- soft, pred, &mask, mergedregs);
+ unsigned pred_delay = delay_calc(pred, NULL, consumer, distance,
+ &mask, mergedregs);
delay = MAX2(delay, pred_delay);
}
@@ -314,38 +331,13 @@ delay_calc_postra(struct ir3_block *block, struct ir3_instruction *start,
return delay;
}
-/**
- * Calculate delay for post-RA scheduling based on physical registers but not
- * exact (i.e. don't recurse into predecessors, and make it possible to
- * estimate impact of sync flags).
- *
- * @soft: If true, add additional delay for situations where they
- * would not be strictly required because a sync flag would be
- * used (but scheduler would prefer to schedule some other
- * instructions first to avoid stalling on sync flag)
- * @mergedregs: True if mergedregs is enabled.
- */
-unsigned
-ir3_delay_calc_postra(struct ir3_block *block, struct ir3_instruction *instr,
- bool soft, bool mergedregs)
-{
- regmask_t mask;
- regmask_init(&mask, mergedregs);
- foreach_src (src, instr) {
- if (!(src->flags & (IR3_REG_IMMED | IR3_REG_CONST)))
- regmask_set(&mask, src);
- }
-
- return delay_calc_postra(block, NULL, instr, 0, soft, false, &mask, mergedregs);
-}
-
/**
* Calculate delay for nop insertion. This must exactly match hardware
* requirements, including recursing into predecessor blocks.
*/
unsigned
-ir3_delay_calc_exact(struct ir3_block *block, struct ir3_instruction *instr,
- bool mergedregs)
+ir3_delay_calc(struct ir3_block *block, struct ir3_instruction *instr,
+ bool mergedregs)
{
regmask_t mask;
regmask_init(&mask, mergedregs);
@@ -354,7 +346,7 @@ ir3_delay_calc_exact(struct ir3_block *block, struct ir3_instruction *instr,
regmask_set(&mask, src);
}
- return delay_calc_postra(block, NULL, instr, 0, false, true, &mask, mergedregs);
+ return delay_calc(block, NULL, instr, 0, &mask, mergedregs);
}
/**
diff --git a/src/freedreno/ir3/ir3_legalize.c b/src/freedreno/ir3/ir3_legalize.c
index 42145b2a114..3919c09a8ed 100644
--- a/src/freedreno/ir3/ir3_legalize.c
+++ b/src/freedreno/ir3/ir3_legalize.c
@@ -804,7 +804,7 @@ nop_sched(struct ir3 *ir, struct ir3_shader_variant *so)
list_inithead(&block->instr_list);
foreach_instr_safe (instr, &instr_list) {
- unsigned delay = ir3_delay_calc_exact(block, instr, so->mergedregs);
+ unsigned delay = ir3_delay_calc(block, instr, so->mergedregs);
/* NOTE: I think the nopN encoding works for a5xx and
* probably a4xx, but not a3xx. So far only tested on
diff --git a/src/freedreno/ir3/ir3_postsched.c b/src/freedreno/ir3/ir3_postsched.c
index fa2b7100c16..f30b5b0d454 100644
--- a/src/freedreno/ir3/ir3_postsched.c
+++ b/src/freedreno/ir3/ir3_postsched.c
@@ -68,6 +68,8 @@ struct ir3_postsched_ctx {
struct list_head unscheduled_list; /* unscheduled instructions */
+ unsigned ip;
+
int sfu_delay;
int tex_delay;
};
@@ -77,6 +79,8 @@ struct ir3_postsched_node {
struct ir3_instruction *instr;
bool partially_evaluated_path;
+ unsigned earliest_ip;
+
bool has_tex_src, has_sfu_src;
unsigned delay;
@@ -111,9 +115,26 @@ schedule(struct ir3_postsched_ctx *ctx, struct ir3_instruction *instr)
di(instr, "schedule");
- list_addtail(&instr->node, &instr->block->instr_list);
+ bool counts_for_delay = is_alu(instr) || is_flow(instr);
+
+ unsigned delay_cycles = counts_for_delay ? 1 + instr->repeat : 0;
struct ir3_postsched_node *n = instr->data;
+
+ /* We insert any nop's needed to get to earliest_ip, then advance
+ * delay_cycles by scheduling the instruction.
+ */
+ ctx->ip = MAX2(ctx->ip, n->earliest_ip) + delay_cycles;
+
+ util_dynarray_foreach (&n->dag.edges, struct dag_edge, edge) {
+ unsigned delay = (unsigned)(uintptr_t)edge->data;
+ struct ir3_postsched_node *child =
+ container_of(edge->child, struct ir3_postsched_node, dag);
+ child->earliest_ip = MAX2(child->earliest_ip, ctx->ip + delay);
+ }
+
+ list_addtail(&instr->node, &instr->block->instr_list);
+
dag_prune_head(ctx->dag, &n->dag);
if (is_meta(instr) && (instr->opc != OPC_META_TEX_PREFETCH))
@@ -157,7 +178,7 @@ dump_state(struct ir3_postsched_ctx *ctx)
static unsigned
node_delay(struct ir3_postsched_ctx *ctx, struct ir3_postsched_node *n)
{
- return ir3_delay_calc_postra(ctx->block, n->instr, false, ctx->v->mergedregs);
+ return MAX2(n->earliest_ip, ctx->ip) - ctx->ip;
}
static unsigned
@@ -339,6 +360,7 @@ struct ir3_postsched_deps_state {
* for full precision and 2nd half for half-precision.
*/
struct ir3_postsched_node *regs[2 * 256];
+ unsigned dst_n[2 * 256];
};
/* bounds checking read/write accessors, since OoB access to stuff on
@@ -352,7 +374,8 @@ struct ir3_postsched_deps_state {
static void
add_dep(struct ir3_postsched_deps_state *state,
- struct ir3_postsched_node *before, struct ir3_postsched_node *after)
+ struct ir3_postsched_node *before, struct ir3_postsched_node *after,
+ unsigned d)
{
if (!before || !after)
return;
@@ -360,30 +383,36 @@ add_dep(struct ir3_postsched_deps_state *state,
assert(before != after);
if (state->direction == F) {
- dag_add_edge(&before->dag, &after->dag, 0);
+ dag_add_edge_max_data(&before->dag, &after->dag, (uintptr_t)d);
} else {
- dag_add_edge(&after->dag, &before->dag, 0);
+ dag_add_edge_max_data(&after->dag, &before->dag, 0);
}
}
static void
add_single_reg_dep(struct ir3_postsched_deps_state *state,
- struct ir3_postsched_node *node, unsigned num, int src_n)
+ struct ir3_postsched_node *node, unsigned num, int src_n,
+ int dst_n)
{
struct ir3_postsched_node *dep = dep_reg(state, num);
+ unsigned d = 0;
if (src_n >= 0 && dep && state->direction == F) {
- unsigned d = ir3_delayslots(dep->instr, node->instr, src_n, true);
- node->delay = MAX2(node->delay, d);
+ /* get the dst_n this corresponds to */
+ unsigned dst_n = state->dst_n[num];
+ unsigned d_soft = ir3_delayslots(dep->instr, node->instr, src_n, true);
+ d = ir3_delayslots_with_repeat(dep->instr, node->instr, dst_n, src_n);
+ node->delay = MAX2(node->delay, d_soft);
if (is_tex_or_prefetch(dep->instr))
node->has_tex_src = true;
if (is_sfu(dep->instr))
node->has_sfu_src = true;
}
- add_dep(state, dep, node);
+ add_dep(state, dep, node, d);
if (src_n < 0) {
dep_reg(state, num) = node;
+ state->dst_n[num] = dst_n;
}
}
@@ -391,15 +420,15 @@ add_single_reg_dep(struct ir3_postsched_deps_state *state,
* between half and full precision that result in additional dependencies.
* The 'reg' arg is really just to know half vs full precision.
*
- * If non-negative, then this adds a dependency on a source register, and
+ * If src_n is positive, then this adds a dependency on a source register, and
* src_n is the index passed into ir3_delayslots() for calculating the delay:
- * If positive, corresponds to node->instr->regs[src_n]. If negative, then
- * this is for a destination register.
+ * it corresponds to node->instr->srcs[src_n]. If src_n is negative, then
+ * this is for the destination register corresponding to dst_n.
*/
static void
add_reg_dep(struct ir3_postsched_deps_state *state,
struct ir3_postsched_node *node, const struct ir3_register *reg,
- unsigned num, int src_n)
+ unsigned num, int src_n, int dst_n)
{
if (state->merged) {
/* Make sure that special registers like a0.x that are written as
@@ -408,16 +437,16 @@ add_reg_dep(struct ir3_postsched_deps_state *state,
*/
if ((reg->flags & IR3_REG_HALF) && !is_reg_special(reg)) {
/* single conflict in half-reg space: */
- add_single_reg_dep(state, node, num, src_n);
+ add_single_reg_dep(state, node, num, src_n, dst_n);
} else {
/* two conflicts in half-reg space: */
- add_single_reg_dep(state, node, 2 * num + 0, src_n);
- add_single_reg_dep(state, node, 2 * num + 1, src_n);
+ add_single_reg_dep(state, node, 2 * num + 0, src_n, dst_n);
+ add_single_reg_dep(state, node, 2 * num + 1, src_n, dst_n);
}
} else {
if (reg->flags & IR3_REG_HALF)
num += ARRAY_SIZE(state->regs) / 2;
- add_single_reg_dep(state, node, num, src_n);
+ add_single_reg_dep(state, node, num, src_n, dst_n);
}
}
@@ -435,12 +464,12 @@ calculate_deps(struct ir3_postsched_deps_state *state,
if (reg->flags & IR3_REG_RELATIV) {
/* mark entire array as read: */
for (unsigned j = 0; j < reg->size; j++) {
- add_reg_dep(state, node, reg, reg->array.base + j, i);
+ add_reg_dep(state, node, reg, reg->array.base + j, i, -1);
}
} else {
assert(reg->wrmask >= 1);
u_foreach_bit (b, reg->wrmask) {
- add_reg_dep(state, node, reg, reg->num + b, i);
+ add_reg_dep(state, node, reg, reg->num + b, i, -1);
}
}
}
@@ -448,18 +477,18 @@ calculate_deps(struct ir3_postsched_deps_state *state,
/* And then after we update the state for what this instruction
* wrote:
*/
- foreach_dst (reg, node->instr) {
+ foreach_dst_n (reg, i, node->instr) {
if (reg->wrmask == 0)
continue;
if (reg->flags & IR3_REG_RELATIV) {
/* mark the entire array as written: */
- for (unsigned i = 0; i < reg->size; i++) {
- add_reg_dep(state, node, reg, reg->array.base + i, -1);
+ for (unsigned j = 0; j < reg->size; j++) {
+ add_reg_dep(state, node, reg, reg->array.base + j, -1, i);
}
} else {
assert(reg->wrmask >= 1);
u_foreach_bit (b, reg->wrmask) {
- add_reg_dep(state, node, reg, reg->num + b, -1);
+ add_reg_dep(state, node, reg, reg->num + b, -1, i);
}
}
}
@@ -571,7 +600,7 @@ sched_dag_init(struct ir3_postsched_ctx *ctx)
if (src->block != instr->block)
continue;
- dag_add_edge(&sn->dag, &n->dag, 0);
+ dag_add_edge_max_data(&sn->dag, &n->dag, 0);
}
if (is_input(instr)) {
@@ -580,14 +609,14 @@ sched_dag_init(struct ir3_postsched_ctx *ctx)
util_dynarray_foreach (&inputs, struct ir3_instruction *, instrp) {
struct ir3_instruction *input = *instrp;
struct ir3_postsched_node *in = input->data;
- dag_add_edge(&in->dag, &n->dag, 0);
+ dag_add_edge_max_data(&in->dag, &n->dag, 0);
}
util_dynarray_append(&kills, struct ir3_instruction *, instr);
} else if (is_tex(instr) || is_mem(instr)) {
util_dynarray_foreach (&kills, struct ir3_instruction *, instrp) {
struct ir3_instruction *kill = *instrp;
struct ir3_postsched_node *kn = kill->data;
- dag_add_edge(&kn->dag, &n->dag, 0);
+ dag_add_edge_max_data(&kn->dag, &n->dag, 0);
}
}
}
@@ -655,8 +684,7 @@ sched_block(struct ir3_postsched_ctx *ctx, struct ir3_block *block)
while (!list_is_empty(&ctx->unscheduled_list)) {
struct ir3_instruction *instr = choose_instr(ctx);
- unsigned delay =
- ir3_delay_calc_postra(ctx->block, instr, false, ctx->v->mergedregs);
+ unsigned delay = node_delay(ctx, instr->data);
d("delay=%u", delay);
/* and if we run out of instructions that can be scheduled,
diff --git a/src/freedreno/ir3/tests/delay.c b/src/freedreno/ir3/tests/delay.c
index 4f8e072ef6b..3f15d5dde56 100644
--- a/src/freedreno/ir3/tests/delay.c
+++ b/src/freedreno/ir3/tests/delay.c
@@ -183,7 +183,7 @@ main(int argc, char **argv)
*/
list_delinit(&last->node);
- unsigned n = ir3_delay_calc_exact(block, last, true);
+ unsigned n = ir3_delay_calc(block, last, true);
if (n != test->expected_delay) {
printf("%d: FAIL: Expected delay %u, but got %u, for:\n%s\n", i,
More information about the mesa-commit
mailing list