Mesa (main): ir3/spill: Initial implementation of rematerialization
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Nov 9 00:10:55 UTC 2021
Module: Mesa
Branch: main
Commit: 38f0b36f1ac192e7e44f3218b08e8ea758a98d29
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=38f0b36f1ac192e7e44f3218b08e8ea758a98d29
Author: Connor Abbott <cwabbott0 at gmail.com>
Date: Mon Oct 11 17:02:28 2021 +0200
ir3/spill: Initial implementation of rematerialization
This only handles moves from immedates/constants. The next step would be
to rematerialize ALU instructions whose sources are available.
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/13650>
---
src/freedreno/ir3/ir3_spill.c | 98 +++++++++++++++++++++++++++++++++++++++----
1 file changed, 91 insertions(+), 7 deletions(-)
diff --git a/src/freedreno/ir3/ir3_spill.c b/src/freedreno/ir3/ir3_spill.c
index 43cad0a4366..d28b1c7af9f 100644
--- a/src/freedreno/ir3/ir3_spill.c
+++ b/src/freedreno/ir3/ir3_spill.c
@@ -79,6 +79,9 @@ struct ra_spill_interval {
* - It is a destination and we're making space for destinations.
*/
bool cant_spill;
+
+ /* Whether this interval can be rematerialized. */
+ bool can_rematerialize;
};
struct ra_spill_block_state {
@@ -328,6 +331,49 @@ compute_next_distance(struct ra_spill_ctx *ctx, struct ir3 *ir)
}
}
+static bool
+can_rematerialize(struct ir3_register *reg)
+{
+ if (reg->flags & IR3_REG_ARRAY)
+ return false;
+ if (reg->instr->opc != OPC_MOV)
+ return false;
+ if (!(reg->instr->srcs[0]->flags & (IR3_REG_IMMED | IR3_REG_CONST)))
+ return false;
+ if (reg->instr->srcs[0]->flags & IR3_REG_RELATIV)
+ return false;
+ return true;
+}
+
+static struct ir3_register *
+rematerialize(struct ir3_register *reg, struct ir3_instruction *after,
+ struct ir3_block *block)
+{
+ d("rematerializing ssa_%u:%u", reg->instr->serialno, reg->name);
+
+ struct ir3_instruction *remat =
+ ir3_instr_create(block, reg->instr->opc, 1, reg->instr->srcs_count);
+ struct ir3_register *dst = __ssa_dst(remat);
+ dst->flags |= reg->flags & (IR3_REG_HALF | IR3_REG_ARRAY);
+ for (unsigned i = 0; i < reg->instr->srcs_count; i++) {
+ struct ir3_register *src =
+ ir3_src_create(remat, INVALID_REG, reg->instr->srcs[i]->flags);
+ *src = *reg->instr->srcs[i];
+ }
+
+ remat->cat1 = reg->instr->cat1;
+
+ dst->merge_set = reg->merge_set;
+ dst->merge_set_offset = reg->merge_set_offset;
+ dst->interval_start = reg->interval_start;
+ dst->interval_end = reg->interval_end;
+
+ if (after)
+ ir3_instr_move_before(remat, after);
+
+ return dst;
+}
+
static void
ra_spill_interval_init(struct ra_spill_interval *interval,
struct ir3_register *reg)
@@ -338,6 +384,7 @@ ra_spill_interval_init(struct ra_spill_interval *interval,
interval->already_spilled = false;
interval->needs_reload = false;
interval->cant_spill = false;
+ interval->can_rematerialize = can_rematerialize(reg);
}
static struct ra_spill_interval *
@@ -361,6 +408,19 @@ ir3_reg_ctx_to_ctx(struct ir3_reg_ctx *ctx)
return rb_node_data(struct ra_spill_ctx, ctx, reg_ctx);
}
+static int
+spill_interval_cmp(const struct ra_spill_interval *a,
+ const struct ra_spill_interval *b)
+{
+ /* Prioritize intervals that we can rematerialize. */
+ if (a->can_rematerialize && !b->can_rematerialize)
+ return 1;
+ if (!a->can_rematerialize && b->can_rematerialize)
+ return -1;
+
+ return a->next_use_distance - b->next_use_distance;
+}
+
static int
ra_spill_interval_cmp(const struct rb_node *_a, const struct rb_node *_b)
{
@@ -368,7 +428,7 @@ ra_spill_interval_cmp(const struct rb_node *_a, const struct rb_node *_b)
rb_node_data(const struct ra_spill_interval, _a, node);
const struct ra_spill_interval *b =
rb_node_data(const struct ra_spill_interval, _b, node);
- return a->next_use_distance - b->next_use_distance;
+ return spill_interval_cmp(a, b);
}
static int
@@ -378,7 +438,7 @@ ra_spill_interval_half_cmp(const struct rb_node *_a, const struct rb_node *_b)
rb_node_data(const struct ra_spill_interval, _a, half_node);
const struct ra_spill_interval *b =
rb_node_data(const struct ra_spill_interval, _b, half_node);
- return a->next_use_distance - b->next_use_distance;
+ return spill_interval_cmp(a, b);
}
static void
@@ -482,8 +542,16 @@ init_dst(struct ra_spill_ctx *ctx, struct ir3_register *dst)
{
struct ra_spill_interval *interval = ctx->intervals[dst->name];
ra_spill_interval_init(interval, dst);
- if (ctx->spilling)
+ if (ctx->spilling) {
interval->next_use_distance = dst->next_use;
+
+ /* We only need to keep track of used-ness if this value may be
+ * rematerialized. This also keeps us from nuking things that may be
+ * in the keeps list (e.g. atomics, input splits).
+ */
+ if (interval->can_rematerialize)
+ dst->instr->flags |= IR3_INSTR_UNUSED;
+ }
}
static void
@@ -631,6 +699,7 @@ set_src_val(struct ir3_register *src, const struct reg_or_immed *val)
src->def = NULL;
} else {
src->def = val->def;
+ val->def->instr->flags &= ~IR3_INSTR_UNUSED;
}
}
@@ -665,6 +734,7 @@ spill(struct ra_spill_ctx *ctx, const struct reg_or_immed *val,
reg = materialize_pcopy_src(val, instr, block);
} else {
reg = val->def;
+ reg->instr->flags &= ~IR3_INSTR_UNUSED;
}
d("spilling ssa_%u:%u to %u", reg->instr->serialno, reg->name,
@@ -699,6 +769,9 @@ static void
spill_interval(struct ra_spill_ctx *ctx, struct ra_spill_interval *interval,
struct ir3_instruction *instr, struct ir3_block *block)
{
+ if (interval->can_rematerialize && !interval->interval.reg->merge_set)
+ return;
+
spill(ctx, &interval->dst, get_spill_slot(ctx, interval->interval.reg),
instr, block);
}
@@ -901,7 +974,11 @@ reload_def(struct ra_spill_ctx *ctx, struct ir3_register *def,
}
}
- struct ir3_register *dst = reload(ctx, def, instr, block);
+ struct ir3_register *dst;
+ if (interval->can_rematerialize)
+ dst = rematerialize(def, instr, block);
+ else
+ dst = reload(ctx, def, instr, block);
rewrite_src_interval(ctx, interval, dst, instr, block);
}
@@ -1340,7 +1417,9 @@ spill_live_ins(struct ra_spill_ctx *ctx, struct ir3_block *block)
if (all_preds_visited &&
is_live_in_all_preds(ctx, interval->interval.reg, block))
continue;
- spill_live_in(ctx, interval->interval.reg, block);
+ if (interval->interval.reg->merge_set ||
+ !interval->can_rematerialize)
+ spill_live_in(ctx, interval->interval.reg, block);
ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
if (ctx->cur_pressure.half <= ctx->limit_pressure.half)
break;
@@ -1410,7 +1489,10 @@ reload_live_in(struct ra_spill_ctx *ctx, struct ir3_register *def,
if (!new_val) {
new_val = ralloc(ctx, struct reg_or_immed);
- new_val->def = reload(ctx, def, NULL, pred);
+ if (interval->can_rematerialize)
+ new_val->def = rematerialize(def, NULL, pred);
+ else
+ new_val->def = reload(ctx, def, NULL, pred);
new_val->flags = new_val->def->flags;
}
live_in_rewrite(ctx, interval, new_val, block, i);
@@ -1557,7 +1639,9 @@ spill_live_out(struct ra_spill_ctx *ctx, struct ra_spill_interval *interval,
{
struct ir3_register *def = interval->interval.reg;
- spill(ctx, &interval->dst, get_spill_slot(ctx, def), NULL, block);
+ if (interval->interval.reg->merge_set ||
+ !interval->can_rematerialize)
+ spill(ctx, &interval->dst, get_spill_slot(ctx, def), NULL, block);
ir3_reg_interval_remove_all(&ctx->reg_ctx, &interval->interval);
}
More information about the mesa-commit
mailing list