Mesa (main): intel/fs: Lower untyped float atomic messages to LSC when available

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: 8f82c8aa1adb0caf0b27d51afd4aa67a36ae8aff
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=8f82c8aa1adb0caf0b27d51afd4aa67a36ae8aff

Author: Sagar Ghuge <sagar.ghuge at intel.com>
Date:   Thu Apr 29 20:50:42 2021 -0700

intel/fs: Lower untyped float atomic messages to LSC when available

Reviewed-by: Jason Ekstrand <jason at jlekstrand.net>
Reviewed-by: Sagar Ghuge <sagar.ghuge at intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11600>

---

 src/intel/compiler/brw_fs.cpp                    | 27 ++++++++++++++++++++----
 src/intel/compiler/brw_ir_performance.cpp        |  5 +++++
 src/intel/compiler/brw_schedule_instructions.cpp |  5 +++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp
index 4ff29a94cc5..da894a8e0af 100644
--- a/src/intel/compiler/brw_fs.cpp
+++ b/src/intel/compiler/brw_fs.cpp
@@ -5878,6 +5878,21 @@ brw_atomic_op_to_lsc_atomic_op(unsigned op)
    }
 }
 
+static enum lsc_opcode
+brw_atomic_op_to_lsc_fatomic_op(uint32_t aop)
+{
+   switch(aop) {
+   case BRW_AOP_FMAX:
+      return LSC_OP_ATOMIC_FMAX;
+   case BRW_AOP_FMIN:
+      return LSC_OP_ATOMIC_FMIN;
+   case BRW_AOP_FCMPWR:
+      return LSC_OP_ATOMIC_FCMPXCHG;
+   default:
+      unreachable("Unsupported float atomic opcode");
+   }
+}
+
 static void
 lower_lsc_surface_logical_send(const fs_builder &bld, fs_inst *inst)
 {
@@ -5952,14 +5967,17 @@ lower_lsc_surface_logical_send(const fs_builder &bld, fs_inst *inst)
                                 false /* has_dest */);
       break;
    case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
+   case SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL: {
       /* Bspec: Atomic instruction -> Cache section:
        *
        *    Atomic messages are always forced to "un-cacheable" in the L1
        *    cache.
        */
-      inst->desc = lsc_msg_desc(devinfo,
-                                brw_atomic_op_to_lsc_atomic_op(arg.ud),
-                                inst->exec_size,
+      enum lsc_opcode opcode =
+         inst->opcode == SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL ?
+         brw_atomic_op_to_lsc_fatomic_op(arg.ud) :
+         brw_atomic_op_to_lsc_atomic_op(arg.ud);
+      inst->desc = lsc_msg_desc(devinfo, opcode, inst->exec_size,
                                 surf_type, LSC_ADDR_SIZE_A32,
                                 1 /* num_coordinates */,
                                 LSC_DATA_SIZE_D32, 1 /* num_channels */,
@@ -5967,6 +5985,7 @@ lower_lsc_surface_logical_send(const fs_builder &bld, fs_inst *inst)
                                 LSC_CACHE_STORE_L1UC_L3WB,
                                 !inst->dst.is_null());
       break;
+   }
    default:
       unreachable("Unknown surface logical instruction");
    }
@@ -6583,6 +6602,7 @@ fs_visitor::lower_logical_sends()
       case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
       case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
       case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
+      case SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL:
          if (devinfo->has_lsc) {
             lower_lsc_surface_logical_send(ibld, inst);
             break;
@@ -6591,7 +6611,6 @@ fs_visitor::lower_logical_sends()
       case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL:
       case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL:
       case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL:
-      case SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL:
       case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
       case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
       case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL:
diff --git a/src/intel/compiler/brw_ir_performance.cpp b/src/intel/compiler/brw_ir_performance.cpp
index d04514cb431..9ae09325aee 100644
--- a/src/intel/compiler/brw_ir_performance.cpp
+++ b/src/intel/compiler/brw_ir_performance.cpp
@@ -1115,6 +1115,11 @@ namespace {
             case LSC_OP_ATOMIC_UMIN:
             case LSC_OP_ATOMIC_UMAX:
             case LSC_OP_ATOMIC_CMPXCHG:
+            case LSC_OP_ATOMIC_FADD:
+            case LSC_OP_ATOMIC_FSUB:
+            case LSC_OP_ATOMIC_FMIN:
+            case LSC_OP_ATOMIC_FMAX:
+            case LSC_OP_ATOMIC_FCMPXCHG:
             case LSC_OP_ATOMIC_AND:
             case LSC_OP_ATOMIC_OR:
             case LSC_OP_ATOMIC_XOR:
diff --git a/src/intel/compiler/brw_schedule_instructions.cpp b/src/intel/compiler/brw_schedule_instructions.cpp
index b5cd1064c55..b7cd4d8dda8 100644
--- a/src/intel/compiler/brw_schedule_instructions.cpp
+++ b/src/intel/compiler/brw_schedule_instructions.cpp
@@ -547,6 +547,11 @@ schedule_node::set_latency_gfx7(bool is_haswell)
          case LSC_OP_ATOMIC_UMIN:
          case LSC_OP_ATOMIC_UMAX:
          case LSC_OP_ATOMIC_CMPXCHG:
+         case LSC_OP_ATOMIC_FADD:
+         case LSC_OP_ATOMIC_FSUB:
+         case LSC_OP_ATOMIC_FMIN:
+         case LSC_OP_ATOMIC_FMAX:
+         case LSC_OP_ATOMIC_FCMPXCHG:
          case LSC_OP_ATOMIC_AND:
          case LSC_OP_ATOMIC_OR:
          case LSC_OP_ATOMIC_XOR: