[Mesa-dev] [PATCH v2 38/42] nir: Add nir intrinsics for shared variable atomic operations

[Iago Toral]

On Tue, 2015-11-17 at 21:55 -0800, Jordan Justen wrote:
> Signed-off-by: Jordan Justen <jordan.l.justen at intel.com>
> ---
>  src/glsl/nir/glsl_to_nir.cpp  | 53 +++++++++++++++++++++++++++++++++++++++++++
>  src/glsl/nir/nir_intrinsics.h | 25 ++++++++++++++++++++
>  2 files changed, 78 insertions(+)
> 
> diff --git a/src/glsl/nir/glsl_to_nir.cpp b/src/glsl/nir/glsl_to_nir.cpp
> index 83724d3..a7ee4be 100644
> --- a/src/glsl/nir/glsl_to_nir.cpp
> +++ b/src/glsl/nir/glsl_to_nir.cpp
> @@ -731,6 +731,22 @@ nir_visitor::visit(ir_call *ir)
>           op = nir_intrinsic_load_shared;
>        } else if (strcmp(ir->callee_name(), "__intrinsic_store_shared") == 0) {
>           op = nir_intrinsic_store_shared;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_add;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_and;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_or;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_xor;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_min;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_max;

This is based on our early ssbo nir code. However, For min/max we need
to do something different because we need to differentiate between
unsigned and signed versions and our initial nir port did not get that
right. See the nir ssbo implementation in master.

Iago

> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_exchange;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_comp_swap;
>        } else {
>           unreachable("not reached");
>        }
> @@ -1036,6 +1052,43 @@ nir_visitor::visit(ir_call *ir)
>           nir_builder_instr_insert(&b, &instr->instr);
>           break;
>        }
> +      case nir_intrinsic_shared_atomic_add:
> +      case nir_intrinsic_shared_atomic_min:
> +      case nir_intrinsic_shared_atomic_max:
> +      case nir_intrinsic_shared_atomic_and:
> +      case nir_intrinsic_shared_atomic_or:
> +      case nir_intrinsic_shared_atomic_xor:
> +      case nir_intrinsic_shared_atomic_exchange:
> +      case nir_intrinsic_shared_atomic_comp_swap: {
> +         int param_count = ir->actual_parameters.length();
> +         assert(param_count == 2 || param_count == 3);
> +
> +         /* Offset */
> +         exec_node *param = ir->actual_parameters.get_head();
> +         ir_instruction *inst = (ir_instruction *) param;
> +         instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +
> +         /* data1 parameter (this is always present) */
> +         param = param->get_next();
> +         inst = (ir_instruction *) param;
> +         instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +
> +         /* data2 parameter (only with atomic_comp_swap) */
> +         if (param_count == 3) {
> +            assert(op == nir_intrinsic_shared_atomic_comp_swap);
> +            param = param->get_next();
> +            inst = (ir_instruction *) param;
> +            instr->src[2] =
> +               nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +         }
> +
> +         /* Atomic result */
> +         assert(ir->return_deref);
> +         nir_ssa_dest_init(&instr->instr, &instr->dest,
> +                           ir->return_deref->type->vector_elements, NULL);
> +         nir_builder_instr_insert(&b, &instr->instr);
> +         break;
> +      }
>        default:
>           unreachable("not reached");
>        }
> diff --git a/src/glsl/nir/nir_intrinsics.h b/src/glsl/nir/nir_intrinsics.h
> index 6912258..31b83fe 100644
> --- a/src/glsl/nir/nir_intrinsics.h
> +++ b/src/glsl/nir/nir_intrinsics.h
> @@ -203,6 +203,31 @@ INTRINSIC(ssbo_atomic_xor, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
>  INTRINSIC(ssbo_atomic_exchange, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
>  INTRINSIC(ssbo_atomic_comp_swap, 4, ARR(1, 1, 1, 1), true, 1, 0, 0, 0)
>  
> +/*
> + * CS shared variable atomic intrinsics
> + *
> + * All of the shared variable atomic memory operations read a value from
> + * memory, compute a new value using one of the operations below, write the
> + * new value to memory, and return the original value read.
> + *
> + * All operations take 2 sources except CompSwap that takes 3. These
> + * sources represent:
> + *
> + * 0: The offset into the shared variable storage region that the atomic
> + *    operation will operate on.
> + * 1: The data parameter to the atomic function (i.e. the value to add
> + *    in shared_atomic_add, etc).
> + * 2: For CompSwap only: the second data parameter.
> + */
> +INTRINSIC(shared_atomic_add, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_min, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_max, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_and, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_or, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_xor, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_exchange, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_comp_swap, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
> +
>  #define SYSTEM_VALUE(name, components, num_indices) \
>     INTRINSIC(load_##name, 0, ARR(), true, components, 0, num_indices, \
>     NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)