[Mesa-dev] [PATCH v3 40/44] nir: Add nir intrinsics for shared variable atomic operations

[Iago Toral]

Reviewed-by: Iago Toral Quiroga <itoral at igalia.com>

On Tue, 2015-12-01 at 00:19 -0800, Jordan Justen wrote:
> v3:
>  * Update min/max based on latest SSBO code (Iago)
> 
> Signed-off-by: Jordan Justen <jordan.l.justen at intel.com>
> Cc: Iago Toral Quiroga <itoral at igalia.com>
> ---
>  src/glsl/nir/glsl_to_nir.cpp  | 67 +++++++++++++++++++++++++++++++++++++++++++
>  src/glsl/nir/nir_intrinsics.h | 27 +++++++++++++++++
>  2 files changed, 94 insertions(+)
> 
> diff --git a/src/glsl/nir/glsl_to_nir.cpp b/src/glsl/nir/glsl_to_nir.cpp
> index 42f0ca5..fc0f404 100644
> --- a/src/glsl/nir/glsl_to_nir.cpp
> +++ b/src/glsl/nir/glsl_to_nir.cpp
> @@ -731,6 +731,34 @@ nir_visitor::visit(ir_call *ir)
>           op = nir_intrinsic_load_shared;
>        } else if (strcmp(ir->callee_name(), "__intrinsic_store_shared") == 0) {
>           op = nir_intrinsic_store_shared;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_add;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_and;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_or;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_xor;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_shared") == 0) {
> +         assert(ir->return_deref);
> +         if (ir->return_deref->type == glsl_type::int_type)
> +            op = nir_intrinsic_shared_atomic_imin;
> +         else if (ir->return_deref->type == glsl_type::uint_type)
> +            op = nir_intrinsic_shared_atomic_umin;
> +         else
> +            unreachable("Invalid type");
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_shared") == 0) {
> +         assert(ir->return_deref);
> +         if (ir->return_deref->type == glsl_type::int_type)
> +            op = nir_intrinsic_shared_atomic_imax;
> +         else if (ir->return_deref->type == glsl_type::uint_type)
> +            op = nir_intrinsic_shared_atomic_umax;
> +         else
> +            unreachable("Invalid type");
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_exchange;
> +      } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_shared") == 0) {
> +         op = nir_intrinsic_shared_atomic_comp_swap;
>        } else {
>           unreachable("not reached");
>        }
> @@ -1036,6 +1064,45 @@ nir_visitor::visit(ir_call *ir)
>           nir_builder_instr_insert(&b, &instr->instr);
>           break;
>        }
> +      case nir_intrinsic_shared_atomic_add:
> +      case nir_intrinsic_shared_atomic_imin:
> +      case nir_intrinsic_shared_atomic_umin:
> +      case nir_intrinsic_shared_atomic_imax:
> +      case nir_intrinsic_shared_atomic_umax:
> +      case nir_intrinsic_shared_atomic_and:
> +      case nir_intrinsic_shared_atomic_or:
> +      case nir_intrinsic_shared_atomic_xor:
> +      case nir_intrinsic_shared_atomic_exchange:
> +      case nir_intrinsic_shared_atomic_comp_swap: {
> +         int param_count = ir->actual_parameters.length();
> +         assert(param_count == 2 || param_count == 3);
> +
> +         /* Offset */
> +         exec_node *param = ir->actual_parameters.get_head();
> +         ir_instruction *inst = (ir_instruction *) param;
> +         instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +
> +         /* data1 parameter (this is always present) */
> +         param = param->get_next();
> +         inst = (ir_instruction *) param;
> +         instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +
> +         /* data2 parameter (only with atomic_comp_swap) */
> +         if (param_count == 3) {
> +            assert(op == nir_intrinsic_shared_atomic_comp_swap);
> +            param = param->get_next();
> +            inst = (ir_instruction *) param;
> +            instr->src[2] =
> +               nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
> +         }
> +
> +         /* Atomic result */
> +         assert(ir->return_deref);
> +         nir_ssa_dest_init(&instr->instr, &instr->dest,
> +                           ir->return_deref->type->vector_elements, NULL);
> +         nir_builder_instr_insert(&b, &instr->instr);
> +         break;
> +      }
>        default:
>           unreachable("not reached");
>        }
> diff --git a/src/glsl/nir/nir_intrinsics.h b/src/glsl/nir/nir_intrinsics.h
> index cf9aa88..6b6cb32 100644
> --- a/src/glsl/nir/nir_intrinsics.h
> +++ b/src/glsl/nir/nir_intrinsics.h
> @@ -203,6 +203,33 @@ INTRINSIC(ssbo_atomic_xor, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
>  INTRINSIC(ssbo_atomic_exchange, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
>  INTRINSIC(ssbo_atomic_comp_swap, 4, ARR(1, 1, 1, 1), true, 1, 0, 0, 0)
>  
> +/*
> + * CS shared variable atomic intrinsics
> + *
> + * All of the shared variable atomic memory operations read a value from
> + * memory, compute a new value using one of the operations below, write the
> + * new value to memory, and return the original value read.
> + *
> + * All operations take 2 sources except CompSwap that takes 3. These
> + * sources represent:
> + *
> + * 0: The offset into the shared variable storage region that the atomic
> + *    operation will operate on.
> + * 1: The data parameter to the atomic function (i.e. the value to add
> + *    in shared_atomic_add, etc).
> + * 2: For CompSwap only: the second data parameter.
> + */
> +INTRINSIC(shared_atomic_add, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_imin, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_umin, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_imax, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_umax, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_and, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_or, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_xor, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_exchange, 2, ARR(1, 1), true, 1, 0, 0, 0)
> +INTRINSIC(shared_atomic_comp_swap, 3, ARR(1, 1, 1), true, 1, 0, 0, 0)
> +
>  #define SYSTEM_VALUE(name, components, num_indices) \
>     INTRINSIC(load_##name, 0, ARR(), true, components, 0, num_indices, \
>     NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)