Mesa (main): nir/opt_load_store_vectorize: handle task payloads

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: 97b53ad759b5f3b21edb4870779a7ca12e6bca52
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=97b53ad759b5f3b21edb4870779a7ca12e6bca52

Author: Marcin Ślusarz <marcin.slusarz at intel.com>
Date:   Mon Jun 13 14:43:45 2022 +0200

nir/opt_load_store_vectorize: handle task payloads

Reviewed-by: Timur Kristóf <timur.kristof at gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17000>

---

 src/compiler/nir/nir_opt_load_store_vectorize.c | 26 ++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/compiler/nir/nir_opt_load_store_vectorize.c b/src/compiler/nir/nir_opt_load_store_vectorize.c
index 81844b8031f..4b6dc71b224 100644
--- a/src/compiler/nir/nir_opt_load_store_vectorize.c
+++ b/src/compiler/nir/nir_opt_load_store_vectorize.c
@@ -82,6 +82,8 @@ case nir_intrinsic_##op: {\
    STORE(nir_var_mem_shared, shared, -1, 1, -1, 0)
    LOAD(nir_var_mem_global, global, -1, 0, -1)
    STORE(nir_var_mem_global, global, -1, 1, -1, 0)
+   LOAD(nir_var_mem_task_payload, task_payload, -1, 0, -1)
+   STORE(nir_var_mem_task_payload, task_payload, -1, 1, -1, 0)
    ATOMIC(nir_var_mem_ssbo, ssbo, add, 0, 1, -1, 2)
    ATOMIC(nir_var_mem_ssbo, ssbo, imin, 0, 1, -1, 2)
    ATOMIC(nir_var_mem_ssbo, ssbo, umin, 0, 1, -1, 2)
@@ -138,6 +140,20 @@ case nir_intrinsic_##op: {\
    ATOMIC(nir_var_mem_global, global, fmin, -1, 0, -1, 1)
    ATOMIC(nir_var_mem_global, global, fmax, -1, 0, -1, 1)
    ATOMIC(nir_var_mem_global, global, fcomp_swap, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, add, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, imin, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, umin, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, imax, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, umax, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, and, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, or, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, xor, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, exchange, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, comp_swap, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, fadd, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, fmin, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, fmax, -1, 0, -1, 1)
+   ATOMIC(nir_var_mem_task_payload, task_payload, fcomp_swap, -1, 0, -1, 1)
    default:
       break;
 #undef ATOMIC
@@ -611,6 +627,7 @@ create_entry(struct vectorize_ctx *ctx,
    restrict_modes |= nir_var_shader_temp | nir_var_function_temp;
    restrict_modes |= nir_var_uniform | nir_var_mem_push_const;
    restrict_modes |= nir_var_system_value | nir_var_mem_shared;
+   restrict_modes |= nir_var_mem_task_payload;
    if (get_variable_mode(entry) & restrict_modes)
       entry->access |= ACCESS_RESTRICT;
 
@@ -1355,13 +1372,15 @@ handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *imp
       switch (intrin->intrinsic) {
       case nir_intrinsic_group_memory_barrier:
       case nir_intrinsic_memory_barrier:
-         modes = nir_var_mem_ssbo | nir_var_mem_shared | nir_var_mem_global;
+         modes = nir_var_mem_ssbo | nir_var_mem_shared | nir_var_mem_global |
+                 nir_var_mem_task_payload;
          break;
       /* prevent speculative loads/stores */
       case nir_intrinsic_discard_if:
       case nir_intrinsic_discard:
       case nir_intrinsic_terminate_if:
       case nir_intrinsic_terminate:
+      case nir_intrinsic_launch_mesh_workgroups:
          modes = nir_var_all;
          break;
       case nir_intrinsic_demote_if:
@@ -1373,7 +1392,7 @@ handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *imp
          modes = nir_var_mem_ssbo | nir_var_mem_global;
          break;
       case nir_intrinsic_memory_barrier_shared:
-         modes = nir_var_mem_shared;
+         modes = nir_var_mem_shared | nir_var_mem_task_payload;
          break;
       case nir_intrinsic_scoped_barrier:
          if (nir_intrinsic_memory_scope(intrin) == NIR_SCOPE_NONE)
@@ -1381,7 +1400,8 @@ handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *imp
 
          modes = nir_intrinsic_memory_modes(intrin) & (nir_var_mem_ssbo |
                                                        nir_var_mem_shared |
-                                                       nir_var_mem_global);
+                                                       nir_var_mem_global |
+                                                       nir_var_mem_task_payload);
          acquire = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_ACQUIRE;
          release = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_RELEASE;
          switch (nir_intrinsic_memory_scope(intrin)) {