Mesa (main): zink: rewrite atomic ssbo intrinsics as atomic derefs
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Fri Jun 10 22:03:39 UTC 2022
Module: Mesa
Branch: main
Commit: 5a95c6b3282ed540ee42a5bfc1924168b0b3e3b2
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=5a95c6b3282ed540ee42a5bfc1924168b0b3e3b2
Author: Mike Blumenkrantz <michael.blumenkrantz at gmail.com>
Date: Fri Jun 10 11:32:13 2022 -0400
zink: rewrite atomic ssbo intrinsics as atomic derefs
todo--
Reviewed-by: Jason Ekstrand <jason.ekstrand at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16973>
---
.../drivers/zink/nir_to_spirv/nir_to_spirv.c | 56 ++++---------
src/gallium/drivers/zink/zink_compiler.c | 91 +++++++++++++++++++++-
2 files changed, 106 insertions(+), 41 deletions(-)
diff --git a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
index cfbc945e026..94f38d658b9 100644
--- a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
+++ b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c
@@ -215,7 +215,7 @@ get_atomic_op(nir_intrinsic_op op)
{
switch (op) {
#define CASE_ATOMIC_OP(type) \
- case nir_intrinsic_ssbo_atomic_##type: \
+ case nir_intrinsic_deref_atomic_##type: \
case nir_intrinsic_image_deref_atomic_##type: \
case nir_intrinsic_shared_atomic_##type
@@ -2686,37 +2686,15 @@ handle_atomic_op(struct ntv_context *ctx, nir_intrinsic_instr *intr, SpvId ptr,
}
static void
-emit_ssbo_atomic_intrinsic(struct ntv_context *ctx, nir_intrinsic_instr *intr)
+emit_deref_atomic_intrinsic(struct ntv_context *ctx, nir_intrinsic_instr *intr)
{
- SpvId ssbo;
- SpvId param;
- SpvId dest_type = get_dest_type(ctx, &intr->dest, nir_type_uint32);
-
- unsigned bit_size = MIN2(nir_src_bit_size(intr->src[0]), 32);
- unsigned idx = bit_size >> 4;
- assert(idx < ARRAY_SIZE(ctx->ssbos));
- ssbo = ctx->ssbos[idx];
- param = get_src(ctx, &intr->src[2]);
-
- SpvId pointer_type = spirv_builder_type_pointer(&ctx->builder,
- SpvStorageClassStorageBuffer,
- dest_type);
- SpvId uint_type = get_uvec_type(ctx, 32, 1);
- /* an id of the array stride in bytes */
- SpvId uint_size = emit_uint_const(ctx, 32, bit_size / 8);
- SpvId bo = get_src(ctx, &intr->src[0]);
- SpvId member = emit_uint_const(ctx, 32, 0);
- SpvId offset = get_src(ctx, &intr->src[1]);
- SpvId vec_offset = emit_binop(ctx, SpvOpUDiv, uint_type, offset, uint_size);
- SpvId indices[] = { bo, member, vec_offset };
- SpvId ptr = spirv_builder_emit_access_chain(&ctx->builder, pointer_type,
- ssbo, indices,
- ARRAY_SIZE(indices));
+ SpvId ptr = get_src(ctx, &intr->src[0]);
+ SpvId param = get_src(ctx, &intr->src[1]);
SpvId param2 = 0;
- if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
- param2 = get_src(ctx, &intr->src[3]);
+ if (intr->intrinsic == nir_intrinsic_deref_atomic_comp_swap)
+ param2 = get_src(ctx, &intr->src[2]);
handle_atomic_op(ctx, intr, ptr, param, param2, nir_type_uint32);
}
@@ -3191,17 +3169,17 @@ emit_intrinsic(struct ntv_context *ctx, nir_intrinsic_instr *intr)
SpvMemorySemanticsAcquireReleaseMask);
break;
- case nir_intrinsic_ssbo_atomic_add:
- case nir_intrinsic_ssbo_atomic_umin:
- case nir_intrinsic_ssbo_atomic_imin:
- case nir_intrinsic_ssbo_atomic_umax:
- case nir_intrinsic_ssbo_atomic_imax:
- case nir_intrinsic_ssbo_atomic_and:
- case nir_intrinsic_ssbo_atomic_or:
- case nir_intrinsic_ssbo_atomic_xor:
- case nir_intrinsic_ssbo_atomic_exchange:
- case nir_intrinsic_ssbo_atomic_comp_swap:
- emit_ssbo_atomic_intrinsic(ctx, intr);
+ case nir_intrinsic_deref_atomic_add:
+ case nir_intrinsic_deref_atomic_umin:
+ case nir_intrinsic_deref_atomic_imin:
+ case nir_intrinsic_deref_atomic_umax:
+ case nir_intrinsic_deref_atomic_imax:
+ case nir_intrinsic_deref_atomic_and:
+ case nir_intrinsic_deref_atomic_or:
+ case nir_intrinsic_deref_atomic_xor:
+ case nir_intrinsic_deref_atomic_exchange:
+ case nir_intrinsic_deref_atomic_comp_swap:
+ emit_deref_atomic_intrinsic(ctx, intr);
break;
case nir_intrinsic_shared_atomic_add:
diff --git a/src/gallium/drivers/zink/zink_compiler.c b/src/gallium/drivers/zink/zink_compiler.c
index ff0ed5ba851..18567f3400e 100644
--- a/src/gallium/drivers/zink/zink_compiler.c
+++ b/src/gallium/drivers/zink/zink_compiler.c
@@ -940,6 +940,21 @@ rewrite_bo_access_instr(nir_builder *b, nir_instr *instr, void *data)
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
b->cursor = nir_before_instr(instr);
switch (intr->intrinsic) {
+ case nir_intrinsic_ssbo_atomic_add:
+ case nir_intrinsic_ssbo_atomic_umin:
+ case nir_intrinsic_ssbo_atomic_imin:
+ case nir_intrinsic_ssbo_atomic_umax:
+ case nir_intrinsic_ssbo_atomic_imax:
+ case nir_intrinsic_ssbo_atomic_and:
+ case nir_intrinsic_ssbo_atomic_or:
+ case nir_intrinsic_ssbo_atomic_xor:
+ case nir_intrinsic_ssbo_atomic_exchange:
+ case nir_intrinsic_ssbo_atomic_comp_swap: {
+ /* convert offset to uint[idx] */
+ nir_ssa_def *offset = nir_udiv_imm(b, intr->src[1].ssa, 4);
+ nir_instr_rewrite_src_ssa(instr, &intr->src[1], offset);
+ return true;
+ }
case nir_intrinsic_load_ssbo:
case nir_intrinsic_load_ubo: {
/* ubo0 can have unaligned 64bit loads, particularly for bindless texture ids */
@@ -1094,6 +1109,79 @@ get_bo_var(nir_shader *shader, struct bo_vars *bo, bool ssbo, nir_src *src, unsi
return var;
}
+static void
+rewrite_atomic_ssbo_instr(nir_builder *b, nir_instr *instr, struct bo_vars *bo)
+{
+ nir_intrinsic_op op;
+ nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+ switch (intr->intrinsic) {
+ case nir_intrinsic_ssbo_atomic_add:
+ op = nir_intrinsic_deref_atomic_add;
+ break;
+ case nir_intrinsic_ssbo_atomic_umin:
+ op = nir_intrinsic_deref_atomic_umin;
+ break;
+ case nir_intrinsic_ssbo_atomic_imin:
+ op = nir_intrinsic_deref_atomic_imin;
+ break;
+ case nir_intrinsic_ssbo_atomic_umax:
+ op = nir_intrinsic_deref_atomic_umax;
+ break;
+ case nir_intrinsic_ssbo_atomic_imax:
+ op = nir_intrinsic_deref_atomic_imax;
+ break;
+ case nir_intrinsic_ssbo_atomic_and:
+ op = nir_intrinsic_deref_atomic_and;
+ break;
+ case nir_intrinsic_ssbo_atomic_or:
+ op = nir_intrinsic_deref_atomic_or;
+ break;
+ case nir_intrinsic_ssbo_atomic_xor:
+ op = nir_intrinsic_deref_atomic_xor;
+ break;
+ case nir_intrinsic_ssbo_atomic_exchange:
+ op = nir_intrinsic_deref_atomic_exchange;
+ break;
+ case nir_intrinsic_ssbo_atomic_comp_swap:
+ op = nir_intrinsic_deref_atomic_comp_swap;
+ break;
+ default:
+ unreachable("unknown intrinsic");
+ }
+ /* atomic ops are always 32bit */
+ assert(nir_dest_bit_size(intr->dest) == 32);
+ nir_ssa_def *offset = intr->src[1].ssa;
+ nir_src *src = &intr->src[0];
+ nir_variable *var = get_bo_var(b->shader, bo, true, src, 32);
+ nir_deref_instr *deref_var = nir_build_deref_var(b, var);
+ nir_ssa_def *idx = src->ssa;
+ if (bo->first_ssbo)
+ idx = nir_iadd_imm(b, idx, -bo->first_ssbo);
+ nir_deref_instr *deref_array = nir_build_deref_array(b, deref_var, idx);
+ nir_deref_instr *deref_struct = nir_build_deref_struct(b, deref_array, 0);
+
+ /* generate new atomic deref ops for every component */
+ nir_ssa_def *result[4];
+ unsigned num_components = nir_dest_num_components(intr->dest);
+ for (unsigned i = 0; i < num_components; i++) {
+ nir_deref_instr *deref_arr = nir_build_deref_array(b, deref_struct, offset);
+ nir_intrinsic_instr *new_instr = nir_intrinsic_instr_create(b->shader, op);
+ nir_ssa_dest_init(&new_instr->instr, &new_instr->dest, 1, 32, "");
+ new_instr->src[0] = nir_src_for_ssa(&deref_arr->dest.ssa);
+ /* deref ops have no offset src, so copy the srcs after it */
+ for (unsigned i = 2; i < nir_intrinsic_infos[intr->intrinsic].num_srcs; i++)
+ nir_src_copy(&new_instr->src[i - 1], &intr->src[i]);
+ nir_builder_instr_insert(b, &new_instr->instr);
+
+ result[i] = &new_instr->dest.ssa;
+ offset = nir_iadd_imm(b, offset, 1);
+ }
+
+ nir_ssa_def *load = nir_vec(b, result, num_components);
+ nir_ssa_def_rewrite_uses(&intr->dest.ssa, load);
+ nir_instr_remove(instr);
+}
+
static bool
remove_bo_access_instr(nir_builder *b, nir_instr *instr, void *data)
{
@@ -1108,7 +1196,6 @@ remove_bo_access_instr(nir_builder *b, nir_instr *instr, void *data)
nir_src *src;
bool ssbo = true;
switch (intr->intrinsic) {
- /* TODO: these should all be rewritten to use deref intrinsics */
case nir_intrinsic_ssbo_atomic_add:
case nir_intrinsic_ssbo_atomic_umin:
case nir_intrinsic_ssbo_atomic_imin:
@@ -1119,7 +1206,7 @@ remove_bo_access_instr(nir_builder *b, nir_instr *instr, void *data)
case nir_intrinsic_ssbo_atomic_xor:
case nir_intrinsic_ssbo_atomic_exchange:
case nir_intrinsic_ssbo_atomic_comp_swap:
- nir_instr_rewrite_src_ssa(instr, &intr->src[0], nir_iadd_imm(b, intr->src[0].ssa, -bo->first_ssbo));
+ rewrite_atomic_ssbo_instr(b, instr, bo);
return true;
case nir_intrinsic_store_ssbo:
src = &intr->src[1];
More information about the mesa-commit
mailing list