[Mesa-dev] [PATCH 10/18] radeonsi: inline atomic_fetch_args
Marek Olšák
maraeo at gmail.com
Sat Aug 4 07:54:49 UTC 2018
From: Marek Olšák <marek.olsak at amd.com>
---
.../drivers/radeonsi/si_shader_tgsi_mem.c | 125 +++++++-----------
1 file changed, 51 insertions(+), 74 deletions(-)
diff --git a/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c b/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
index f0220881995..4781526b071 100644
--- a/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
+++ b/src/gallium/drivers/radeonsi/si_shader_tgsi_mem.c
@@ -778,75 +778,20 @@ static void store_emit(
* instructions. */
writeonly_memory ||
inst->Memory.Qualifier & (TGSI_MEMORY_COHERENT | TGSI_MEMORY_VOLATILE))
args.cache_policy = ac_glc;
emit_data->output[emit_data->chan] =
ac_build_image_opcode(&ctx->ac, &args);
}
}
-static void atomic_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- const struct tgsi_full_instruction * inst = emit_data->inst;
- LLVMValueRef data1, data2;
- LLVMValueRef rsrc;
- LLVMValueRef tmp;
-
- emit_data->dst_type = ctx->f32;
-
- tmp = lp_build_emit_fetch(bld_base, inst, 2, 0);
- data1 = ac_to_integer(&ctx->ac, tmp);
-
- if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
- tmp = lp_build_emit_fetch(bld_base, inst, 3, 0);
- data2 = ac_to_integer(&ctx->ac, tmp);
- }
-
- /* llvm.amdgcn.image/buffer.atomic.cmpswap reflect the hardware order
- * of arguments, which is reversed relative to TGSI (and GLSL)
- */
- if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS)
- emit_data->args[emit_data->arg_count++] = data2;
- emit_data->args[emit_data->arg_count++] = data1;
-
- if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
- LLVMValueRef offset;
-
- rsrc = shader_buffer_fetch_rsrc(ctx, &inst->Src[0], false);
-
- tmp = lp_build_emit_fetch(bld_base, inst, 1, 0);
- offset = ac_to_integer(&ctx->ac, tmp);
-
- buffer_append_args(ctx, emit_data, rsrc, ctx->i32_0,
- offset, true, false);
- } else if (inst->Src[0].Register.File == TGSI_FILE_IMAGE ||
- tgsi_is_bindless_image_file(inst->Src[0].Register.File)) {
- unsigned target = inst->Memory.Texture;
-
- image_fetch_rsrc(bld_base, &inst->Src[0], true, target, &rsrc);
- image_fetch_coords(bld_base, inst, 1, rsrc,
- &emit_data->args[emit_data->arg_count + 1]);
-
- if (target == TGSI_TEXTURE_BUFFER) {
- buffer_append_args(ctx, emit_data, rsrc,
- emit_data->args[emit_data->arg_count + 1],
- ctx->i32_0, true, false);
- } else {
- emit_data->args[emit_data->arg_count] = rsrc;
- }
- }
-}
-
static void atomic_emit_memory(struct si_shader_context *ctx,
struct lp_build_emit_data *emit_data) {
LLVMBuilderRef builder = ctx->ac.builder;
const struct tgsi_full_instruction * inst = emit_data->inst;
LLVMValueRef ptr, result, arg;
ptr = get_memory_ptr(ctx, inst, ctx->i32, 1);
arg = lp_build_emit_fetch(&ctx->bld_base, inst, 2, 0);
arg = ac_to_integer(&ctx->ac, arg);
@@ -896,43 +841,80 @@ static void atomic_emit_memory(struct si_shader_context *ctx,
op = LLVMAtomicRMWBinOpMax;
break;
default:
unreachable("unknown atomic opcode");
}
result = LLVMBuildAtomicRMW(builder, op, ptr, arg,
LLVMAtomicOrderingSequentiallyConsistent,
false);
}
- emit_data->output[emit_data->chan] = LLVMBuildBitCast(builder, result, emit_data->dst_type, "");
+ emit_data->output[emit_data->chan] =
+ LLVMBuildBitCast(builder, result, ctx->f32, "");
}
static void atomic_emit(
const struct lp_build_tgsi_action *action,
struct lp_build_tgsi_context *bld_base,
struct lp_build_emit_data *emit_data)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
const struct tgsi_full_instruction * inst = emit_data->inst;
- LLVMValueRef tmp;
if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) {
atomic_emit_memory(ctx, emit_data);
return;
}
+ if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
+ /* llvm.amdgcn.image/buffer.atomic.cmpswap reflect the hardware order
+ * of arguments, which is reversed relative to TGSI (and GLSL)
+ */
+ emit_data->args[emit_data->arg_count++] =
+ ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 3, 0));
+ }
+
+ emit_data->args[emit_data->arg_count++] =
+ ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 2, 0));
+
+ if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
+ LLVMValueRef rsrc, offset;
+
+ rsrc = shader_buffer_fetch_rsrc(ctx, &inst->Src[0], false);
+ offset = ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 1, 0));
+
+ buffer_append_args(ctx, emit_data, rsrc, ctx->i32_0,
+ offset, true, false);
+ } else if (inst->Src[0].Register.File == TGSI_FILE_IMAGE ||
+ tgsi_is_bindless_image_file(inst->Src[0].Register.File)) {
+ unsigned target = inst->Memory.Texture;
+ LLVMValueRef rsrc;
+
+ image_fetch_rsrc(bld_base, &inst->Src[0], true, target, &rsrc);
+ image_fetch_coords(bld_base, inst, 1, rsrc,
+ &emit_data->args[emit_data->arg_count + 1]);
+
+ if (target == TGSI_TEXTURE_BUFFER) {
+ buffer_append_args(ctx, emit_data, rsrc,
+ emit_data->args[emit_data->arg_count + 1],
+ ctx->i32_0, true, false);
+ } else {
+ emit_data->args[emit_data->arg_count] = rsrc;
+ }
+ }
+
if (inst->Src[0].Register.File == TGSI_FILE_BUFFER ||
inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
char intrinsic_name[40];
snprintf(intrinsic_name, sizeof(intrinsic_name),
"llvm.amdgcn.buffer.atomic.%s", action->intr_name);
- tmp = ac_build_intrinsic(
+ LLVMValueRef tmp = ac_build_intrinsic(
&ctx->ac, intrinsic_name, ctx->i32,
emit_data->args, emit_data->arg_count, 0);
emit_data->output[emit_data->chan] = ac_to_float(&ctx->ac, tmp);
} else {
unsigned num_data = inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS ? 2 : 1;
struct ac_image_args args = {};
if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
args.opcode = ac_image_atomic_cmpswap;
} else {
@@ -1787,24 +1769,21 @@ static void si_llvm_emit_fbfetch(const struct lp_build_tgsi_action *action,
emit_data->output[emit_data->chan] =
ac_build_image_opcode(&ctx->ac, &args);
}
/**
* Setup actions for TGSI memory opcode, including texture opcodes.
*/
void si_shader_context_init_mem(struct si_shader_context *ctx)
{
- struct lp_build_tgsi_context *bld_base;
- struct lp_build_tgsi_action tmpl = {};
-
- bld_base = &ctx->bld_base;
+ struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
bld_base->op_actions[TGSI_OPCODE_TEX].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TEX_LZ].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TEX2].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXB].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXB2].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXD].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXF].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXF_LZ].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXL].emit = build_tex_intrinsic;
@@ -1814,33 +1793,31 @@ void si_shader_context_init_mem(struct si_shader_context *ctx)
bld_base->op_actions[TGSI_OPCODE_TG4].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_LODQ].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXQS].emit = si_llvm_emit_txqs;
bld_base->op_actions[TGSI_OPCODE_FBFETCH].emit = si_llvm_emit_fbfetch;
bld_base->op_actions[TGSI_OPCODE_LOAD].emit = load_emit;
bld_base->op_actions[TGSI_OPCODE_STORE].emit = store_emit;
bld_base->op_actions[TGSI_OPCODE_RESQ].emit = resq_emit;
- tmpl.fetch_args = atomic_fetch_args;
- tmpl.emit = atomic_emit;
- bld_base->op_actions[TGSI_OPCODE_ATOMUADD] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMUADD].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMUADD].intr_name = "add";
- bld_base->op_actions[TGSI_OPCODE_ATOMXCHG] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMXCHG].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMXCHG].intr_name = "swap";
- bld_base->op_actions[TGSI_OPCODE_ATOMCAS] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMCAS].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMCAS].intr_name = "cmpswap";
- bld_base->op_actions[TGSI_OPCODE_ATOMAND] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMAND].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMAND].intr_name = "and";
- bld_base->op_actions[TGSI_OPCODE_ATOMOR] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMOR].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMOR].intr_name = "or";
- bld_base->op_actions[TGSI_OPCODE_ATOMXOR] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMXOR].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMXOR].intr_name = "xor";
- bld_base->op_actions[TGSI_OPCODE_ATOMUMIN] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMUMIN].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMUMIN].intr_name = "umin";
- bld_base->op_actions[TGSI_OPCODE_ATOMUMAX] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMUMAX].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMUMAX].intr_name = "umax";
- bld_base->op_actions[TGSI_OPCODE_ATOMIMIN] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMIMIN].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMIMIN].intr_name = "smin";
- bld_base->op_actions[TGSI_OPCODE_ATOMIMAX] = tmpl;
+ bld_base->op_actions[TGSI_OPCODE_ATOMIMAX].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMIMAX].intr_name = "smax";
}
--
2.17.1
More information about the mesa-dev
mailing list