[Mesa-dev] [PATCH 07/15] clover/spirv: Add functions for parsing arguments, linking programs, etc.
Karol Herbst
kherbst at redhat.com
Sat May 11 14:07:04 UTC 2019
From: Pierre Moreau <pierre.morrow at free.fr>
v2 (Karol Herbst):
silence warnings about unhandled enum values
---
.../clover/spirv/invocation.cpp | 598 ++++++++++++++++++
.../clover/spirv/invocation.hpp | 12 +
2 files changed, 610 insertions(+)
diff --git a/src/gallium/state_trackers/clover/spirv/invocation.cpp b/src/gallium/state_trackers/clover/spirv/invocation.cpp
index b874f2f061c..62886e77495 100644
--- a/src/gallium/state_trackers/clover/spirv/invocation.cpp
+++ b/src/gallium/state_trackers/clover/spirv/invocation.cpp
@@ -22,10 +22,24 @@
#include "invocation.hpp"
+#include <stack>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
#ifdef CLOVER_ALLOW_SPIRV
#include <spirv-tools/libspirv.hpp>
+#include <spirv-tools/linker.hpp>
#endif
+#include "core/error.hpp"
+#include "core/platform.hpp"
+#include "invocation.hpp"
+#include "llvm/util.hpp"
+#include "pipe/p_state.h"
+#include "util/algorithm.hpp"
+#include "util/functional.hpp"
#include "util/u_math.h"
#include "compiler/spirv/spirv.h"
@@ -34,6 +48,472 @@ using namespace clover;
namespace {
+ template<typename T>
+ T get(const char *source, size_t index) {
+ const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source);
+ return static_cast<T>(word_ptr[index]);
+ }
+
+ enum module::argument::type
+ convertStorageClass(SpvStorageClass storage_class, std::string &err) {
+ switch (storage_class) {
+ case SpvStorageClassFunction:
+ return module::argument::scalar;
+ case SpvStorageClassUniformConstant:
+ return module::argument::constant;
+ case SpvStorageClassWorkgroup:
+ return module::argument::local;
+ case SpvStorageClassCrossWorkgroup:
+ return module::argument::global;
+ default:
+ err += "Invalid storage type " + std::to_string(storage_class) + "\n";
+ throw build_error();
+ }
+ }
+
+ enum module::argument::type
+ convertImageType(SpvId id, SpvDim dim, SpvAccessQualifier access,
+ std::string &err) {
+#define APPEND_DIM(d) \
+ switch(access) { \
+ case SpvAccessQualifierReadOnly: \
+ return module::argument::image##d##_rd; \
+ case SpvAccessQualifierWriteOnly: \
+ return module::argument::image##d##_wr; \
+ default: \
+ err += "Unsupported access qualifier " #d " for image " + \
+ std::to_string(id); \
+ throw build_error(); \
+ }
+
+ switch (dim) {
+ case SpvDim2D:
+ APPEND_DIM(2d)
+ case SpvDim3D:
+ APPEND_DIM(3d)
+ default:
+ err += "Unsupported dimension " + std::to_string(dim) + " for image " +
+ std::to_string(id);
+ throw build_error();
+ }
+
+#undef APPEND_DIM
+ }
+
+ module::section
+ make_text_section(const std::vector<char> &code,
+ enum module::section::type section_type) {
+ const pipe_llvm_program_header header { uint32_t(code.size()) };
+ module::section text { 0, section_type, header.num_bytes, {} };
+
+ text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
+ reinterpret_cast<const char *>(&header) + sizeof(header));
+ text.data.insert(text.data.end(), code.begin(), code.end());
+
+ return text;
+ }
+
+ module
+ create_module_from_spirv(const std::vector<char> &source,
+ size_t pointer_byte_size,
+ std::string &err) {
+ const size_t length = source.size() / sizeof(uint32_t);
+ size_t i = 5u; // Skip header
+
+ std::string kernel_name;
+ size_t kernel_nb = 0u;
+ std::vector<module::argument> args;
+
+ module m;
+
+ std::unordered_map<SpvId, std::string> kernels;
+ std::unordered_map<SpvId, module::argument> types;
+ std::unordered_map<SpvId, SpvId> pointer_types;
+ std::unordered_map<SpvId, unsigned int> constants;
+ std::unordered_set<SpvId> packed_structures;
+ std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>>
+ func_param_attr_map;
+
+#define GET_OPERAND(type, operand_id) get<type>(source.data(), i + operand_id)
+
+ while (i < length) {
+ const auto desc_word = get<uint32_t>(source.data(), i);
+ const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+ const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+ switch (opcode) {
+ case SpvOpEntryPoint:
+ if (GET_OPERAND(SpvExecutionModel, 1) == SpvExecutionModelKernel)
+ kernels.emplace(GET_OPERAND(SpvId, 2),
+ source.data() + (i + 3u) * sizeof(uint32_t));
+ break;
+
+ case SpvOpDecorate: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const auto decoration = GET_OPERAND(SpvDecoration, 2);
+ if (decoration == SpvDecorationCPacked)
+ packed_structures.emplace(id);
+ else if (decoration == SpvDecorationFuncParamAttr)
+ func_param_attr_map[id].push_back(GET_OPERAND(SpvFunctionParameterAttribute, 3u));
+ break;
+ }
+
+ case SpvOpGroupDecorate: {
+ const auto group_id = GET_OPERAND(SpvId, 1);
+ if (packed_structures.count(group_id)) {
+ for (unsigned int i = 2u; i < num_operands; ++i)
+ packed_structures.emplace(GET_OPERAND(SpvId, i));
+ }
+ const auto func_param_attr_iter =
+ func_param_attr_map.find(group_id);
+ if (func_param_attr_iter != func_param_attr_map.end()) {
+ for (unsigned int i = 2u; i < num_operands; ++i)
+ func_param_attr_map.emplace(GET_OPERAND(SpvId, i),
+ func_param_attr_iter->second);
+ }
+ break;
+ }
+
+ case SpvOpConstant:
+ // We only care about constants that represents the size of arrays.
+ // If they are passed as argument, they will never be more than
+ // 4GB-wide, and even if they did, a clover::module::argument size
+ // is represented by an int.
+ constants[GET_OPERAND(SpvId, 2)] = GET_OPERAND(unsigned int, 3u);
+ break;
+
+ case SpvOpTypeInt: // FALLTHROUGH
+ case SpvOpTypeFloat: {
+ const auto size = GET_OPERAND(uint32_t, 2) / 8u;
+ types[GET_OPERAND(SpvId, 1)] = { module::argument::scalar,
+ size, size, size,
+ module::argument::zero_ext };
+ break;
+ }
+
+ case SpvOpTypeArray: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const auto type_id = GET_OPERAND(SpvId, 2);
+ const auto types_iter = types.find(type_id);
+ if (types_iter == types.end())
+ break;
+
+ const auto constant_id = GET_OPERAND(SpvId, 3);
+ const auto constants_iter = constants.find(constant_id);
+ if (constants_iter == constants.end()) {
+ err += "Constant " + std::to_string(constant_id) +
+ " is missing\n";
+ throw build_error();
+ }
+ const auto elem_size = types_iter->second.size;
+ const auto elem_nbs = constants_iter->second;
+ const auto size = elem_size * elem_nbs;
+ types[id] = { module::argument::scalar, size, size,
+ types_iter->second.target_align,
+ module::argument::zero_ext };
+ break;
+ }
+
+ case SpvOpTypeStruct: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const bool is_packed = packed_structures.count(id);
+
+ unsigned struct_size = 0u;
+ unsigned max_elem_size = 0u;
+ for (unsigned j = 2u; j < num_operands; ++j) {
+ const auto type_id = GET_OPERAND(SpvId, j);
+ const auto types_iter = types.find(type_id);
+ if (types_iter == types.end())
+ break;
+
+ const auto alignment = is_packed ? 1u
+ : types_iter->second.target_align;
+ const auto padding = (-struct_size) & (alignment - 1u);
+ struct_size += padding + types_iter->second.target_size;
+ max_elem_size = std::max(max_elem_size,
+ types_iter->second.target_align);
+ }
+ if (!is_packed)
+ struct_size += (-struct_size) & (max_elem_size - 1u);
+
+ types[id] = { module::argument::scalar, struct_size, struct_size,
+ is_packed ? 1u
+ : max_elem_size, module::argument::zero_ext
+ };
+ break;
+ }
+
+ case SpvOpTypeVector: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const auto type_id = GET_OPERAND(SpvId, 2);
+ const auto types_iter = types.find(type_id);
+ if (types_iter == types.end())
+ break;
+
+ const auto elem_size = types_iter->second.size;
+ const auto elem_nbs = GET_OPERAND(uint32_t, 3);
+ const auto size = elem_size * elem_nbs;
+ types[id] = { module::argument::scalar, size, size, size,
+ module::argument::zero_ext };
+ break;
+ }
+
+ case SpvOpTypeForwardPointer: // FALLTHROUGH
+ case SpvOpTypePointer: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const auto storage_class = GET_OPERAND(SpvStorageClass, 2);
+ // Input means this is for a builtin variable, which can not be
+ // passed as an argument to a kernel.
+ if (storage_class == SpvStorageClassInput)
+ break;
+ types[id] = { convertStorageClass(storage_class, err),
+ sizeof(cl_mem),
+ static_cast<module::size_t>(pointer_byte_size),
+ static_cast<module::size_t>(pointer_byte_size),
+ module::argument::zero_ext
+ };
+ if (opcode == SpvOpTypePointer)
+ pointer_types[id] = GET_OPERAND(SpvId, 3);
+ break;
+ }
+
+ case SpvOpTypeSampler:
+ types[GET_OPERAND(SpvId, 1)] = { module::argument::sampler,
+ sizeof(cl_sampler)
+ };
+ break;
+
+ case SpvOpTypeImage: {
+ const auto id = GET_OPERAND(SpvId, 1);
+ const auto dim = GET_OPERAND(SpvDim, 3);
+ const auto access = GET_OPERAND(SpvAccessQualifier, 9);
+ types[id] = { convertImageType(id, dim, access, err),
+ sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem),
+ module::argument::zero_ext
+ };
+ break;
+ }
+
+ case SpvOpTypePipe: // FALLTHROUGH
+ case SpvOpTypeQueue: {
+ err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are "
+ "not available in the currently supported OpenCL C version."
+ "\n";
+ throw build_error();
+ }
+
+ case SpvOpFunction: {
+ const auto kernels_iter = kernels.find(GET_OPERAND(SpvId, 2));
+ if (kernels_iter != kernels.end())
+ kernel_name = kernels_iter->second;
+ break;
+ }
+
+ case SpvOpFunctionParameter: {
+ if (kernel_name.empty())
+ break;
+
+ const auto type_id = GET_OPERAND(SpvId, 1);
+ auto arg = types.find(type_id)->second;
+ const auto &func_param_attr_iter =
+ func_param_attr_map.find(GET_OPERAND(SpvId, 2));
+ if (func_param_attr_iter != func_param_attr_map.end()) {
+ for (auto &i : func_param_attr_iter->second) {
+ switch (i) {
+ case SpvFunctionParameterAttributeSext:
+ arg.ext_type = module::argument::sign_ext;
+ break;
+ case SpvFunctionParameterAttributeZext:
+ arg.ext_type = module::argument::zero_ext;
+ break;
+ case SpvFunctionParameterAttributeByVal: {
+ const SpvId ptr_type_id =
+ pointer_types.find(type_id)->second;
+ arg = types.find(ptr_type_id)->second;
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ }
+ args.emplace_back(arg);
+ break;
+ }
+
+ case SpvOpFunctionEnd:
+ if (kernel_name.empty())
+ break;
+ m.syms.emplace_back(kernel_name, 0, kernel_nb, args);
+ ++kernel_nb;
+ kernel_name.clear();
+ args.clear();
+ break;
+
+ default:
+ break;
+ }
+
+ i += num_operands;
+ }
+
+#undef GET_OPERAND
+
+ m.secs.push_back(make_text_section(source,
+ module::section::text_intermediate));
+ return m;
+ }
+
+ bool
+ check_capabilities(const device &dev, const std::vector<char> &source,
+ std::string &r_log) {
+ const size_t length = source.size() / sizeof(uint32_t);
+ size_t i = 5u; // Skip header
+
+ while (i < length) {
+ const auto desc_word = get<uint32_t>(source.data(), i);
+ const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+ const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+ if (opcode != SpvOpCapability)
+ break;
+
+ const auto capability = get<SpvCapability>(source.data(), i + 1u);
+ switch (capability) {
+ // Mandatory capabilities
+ case SpvCapabilityAddresses:
+ case SpvCapabilityFloat16Buffer:
+ case SpvCapabilityGroups:
+ case SpvCapabilityInt64:
+ case SpvCapabilityInt16:
+ case SpvCapabilityInt8:
+ case SpvCapabilityKernel:
+ case SpvCapabilityLinkage:
+ case SpvCapabilityVector16:
+ break;
+ // Optional capabilities
+ case SpvCapabilityImageBasic:
+ case SpvCapabilityLiteralSampler:
+ case SpvCapabilitySampled1D:
+ case SpvCapabilityImage1D:
+ case SpvCapabilitySampledBuffer:
+ case SpvCapabilityImageBuffer:
+ if (!dev.image_support()) {
+ r_log += "Capability 'ImageBasic' is not supported.\n";
+ return false;
+ }
+ break;
+ case SpvCapabilityFloat64:
+ if (!dev.has_doubles()) {
+ r_log += "Capability 'Float64' is not supported.\n";
+ return false;
+ }
+ break;
+ // Enabled through extensions
+ case SpvCapabilityFloat16:
+ if (!dev.has_halves()) {
+ r_log += "Capability 'Float16' is not supported.\n";
+ return false;
+ }
+ break;
+ default:
+ r_log += "Capability '" + std::to_string(capability) +
+ "' is not supported.\n";
+ return false;
+ }
+
+ i += num_operands;
+ }
+
+ return true;
+ }
+
+ bool
+ check_extensions(const device &dev, const std::vector<char> &source,
+ std::string &r_log) {
+ const size_t length = source.size() / sizeof(uint32_t);
+ size_t i = 5u; // Skip header
+
+ while (i < length) {
+ const auto desc_word = get<uint32_t>(source.data(), i);
+ const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+ const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+ if (opcode == SpvOpCapability) {
+ i += num_operands;
+ continue;
+ }
+ if (opcode != SpvOpExtension)
+ break;
+
+ const char *extension = source.data() + (i + 1u) * sizeof(uint32_t);
+ const std::string device_extensions = dev.supported_extensions();
+ const std::string platform_extensions =
+ dev.platform.supported_extensions();
+ if (device_extensions.find(extension) == std::string::npos &&
+ platform_extensions.find(extension) == std::string::npos) {
+ r_log += "Extension '" + std::string(extension) +
+ "' is not supported.\n";
+ return false;
+ }
+
+ i += num_operands;
+ }
+
+ return true;
+ }
+
+ bool
+ check_memory_model(const device &dev, const std::vector<char> &source,
+ std::string &r_log) {
+ const size_t length = source.size() / sizeof(uint32_t);
+ size_t i = 5u; // Skip header
+
+ while (i < length) {
+ const auto desc_word = get<uint32_t>(source.data(), i);
+ const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+ const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+ switch (opcode) {
+ case SpvOpMemoryModel:
+ switch (get<SpvAddressingModel>(source.data(), i + 1u)) {
+ case SpvAddressingModelPhysical32:
+ return dev.address_bits() == 32;
+ case SpvAddressingModelPhysical64:
+ return dev.address_bits() == 64;
+ default:
+ unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated");
+ return false;
+ }
+ break;
+ default:
+ break;
+ }
+
+ i += num_operands;
+ }
+
+ return false;
+ }
+
+ // Copies the input binary and convert it to the endianness of the host CPU.
+ std::vector<char>
+ spirv_to_cpu(const std::vector<char> &binary)
+ {
+ const uint32_t first_word = get<uint32_t>(binary.data(), 0u);
+ if (first_word == SpvMagicNumber)
+ return binary;
+
+ std::vector<char> cpu_endianness_binary(binary.size());
+ for (size_t i = 0; i < (binary.size() / 4u); ++i) {
+ const uint32_t word = get<uint32_t>(binary.data(), i);
+ reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] =
+ util_bswap32(word);
+ }
+
+ return cpu_endianness_binary;
+ }
+
#ifdef CLOVER_ALLOW_SPIRV
std::string
format_validator_msg(spv_message_level_t level, const char * /* source */,
@@ -96,7 +576,117 @@ clover::spirv::is_binary_spirv(const char *il, size_t length)
(util_bswap32(first_word) == SpvMagicNumber);
}
+module
+clover::spirv::process_program(const std::vector<char> &binary,
+ const device &dev, bool validate,
+ std::string &r_log) {
+ std::vector<char> source = spirv_to_cpu(binary);
+
+ if (validate && !is_valid_spirv(
+ reinterpret_cast<const uint32_t *>(source.data()),
+ source.size() / 4u, dev.device_version(),
+ [&r_log](const char *log){ r_log += std::string(log); }))
+ throw build_error();
+
+ if (!check_capabilities(dev, source, r_log))
+ throw build_error();
+ if (!check_extensions(dev, source, r_log))
+ throw build_error();
+ if (!check_memory_model(dev, source, r_log))
+ throw build_error();
+
+ return create_module_from_spirv(source,
+ dev.address_bits() == 32 ? 4u : 8u, r_log);
+}
+
#ifdef CLOVER_ALLOW_SPIRV
+module
+clover::spirv::link_program(const std::vector<module> &modules,
+ const device &dev, const std::string &opts,
+ std::string &r_log) {
+ std::vector<std::string> options = clover::llvm::tokenize(opts);
+
+ const bool create_library = count("-create-library", options);
+ erase_if(equals("-create-library"), options);
+ erase_if(equals("-enable-link-options"), options);
+
+ if (!options.empty()) {
+ r_log += "SPIR-V linker: Ignoring the following link options: ";
+ for (const auto &opt : options)
+ r_log += "'" + opt + "' ";
+ }
+
+ spvtools::LinkerOptions linker_options;
+ linker_options.SetCreateLibrary(create_library);
+
+ module m;
+
+ const auto section_type = create_library ? module::section::text_library :
+ module::section::text_executable;
+
+ std::vector<const uint32_t *> sections;
+ sections.reserve(modules.size());
+ std::vector<size_t> lengths;
+ lengths.reserve(modules.size());
+
+ auto const validator_consumer = [&r_log](spv_message_level_t level,
+ const char *source,
+ const spv_position_t &position,
+ const char *message) {
+ r_log += format_validator_msg(level, source, position, message);
+ };
+
+ for (const auto &mod : modules) {
+ const module::section *msec = nullptr;
+ try {
+ msec = &find([&](const module::section &sec) {
+ return sec.type == module::section::text_intermediate ||
+ sec.type == module::section::text_library;
+ }, mod.secs);
+ } catch (const std::out_of_range &e) {
+ // We should never reach this as validate_link_devices already checked
+ // that a binary was present.
+ assert(false);
+ }
+
+ const auto c_il = msec->data.data() +
+ sizeof(struct pipe_llvm_program_header);
+ const auto length = msec->size;
+
+ sections.push_back(reinterpret_cast<const uint32_t *>(c_il));
+ lengths.push_back(length / sizeof(uint32_t));
+ }
+
+ std::vector<uint32_t> linked_binary;
+
+ const std::string opencl_version = dev.device_version();
+ const spv_target_env target_env =
+ convert_opencl_str_to_target_env(opencl_version);
+
+ const spvtools::MessageConsumer consumer = validator_consumer;
+ spvtools::Context context(target_env);
+ context.SetMessageConsumer(std::move(consumer));
+
+ if (Link(context, sections.data(), lengths.data(), sections.size(),
+ &linked_binary, linker_options) != SPV_SUCCESS)
+ throw error(CL_LINK_PROGRAM_FAILURE);
+
+ if (!is_valid_spirv(linked_binary.data(), linked_binary.size(),
+ opencl_version,
+ [&r_log](const char *log){ r_log += std::string(log); }))
+ throw error(CL_LINK_PROGRAM_FAILURE);
+
+ for (const auto &mod : modules)
+ m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end());
+
+ m.secs.emplace_back(make_text_section({
+ reinterpret_cast<char *>(linked_binary.data()),
+ reinterpret_cast<char *>(linked_binary.data()) +
+ linked_binary.size() * sizeof(uint32_t) }, section_type));
+
+ return m;
+}
+
bool
clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length,
const std::string &opencl_version,
@@ -120,6 +710,14 @@ clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length,
return spvTool.Validate(binary, length);
}
#else
+module
+clover::spirv::link_program(const std::vector<module> &/*modules*/,
+ const device &/*dev*/, const std::string &/*opts*/,
+ std::string &r_log) {
+ r_log += "SPIRV-Tools and llvm-spirv are required for linking SPIR-V binaries.\n";
+ throw error(CL_LINKER_NOT_AVAILABLE);
+}
+
bool
clover::spirv::is_valid_spirv(const uint32_t * /*binary*/, size_t /*length*/,
const std::string &/*opencl_version*/,
diff --git a/src/gallium/state_trackers/clover/spirv/invocation.hpp b/src/gallium/state_trackers/clover/spirv/invocation.hpp
index 92255a68a8e..37cd1377cb2 100644
--- a/src/gallium/state_trackers/clover/spirv/invocation.hpp
+++ b/src/gallium/state_trackers/clover/spirv/invocation.hpp
@@ -24,6 +24,8 @@
#define CLOVER_SPIRV_INVOCATION_HPP
#include "core/context.hpp"
+#include "core/module.hpp"
+#include "core/program.hpp"
namespace clover {
namespace spirv {
@@ -41,6 +43,16 @@ namespace clover {
bool is_valid_spirv(const uint32_t *binary, size_t length,
const std::string &opencl_version,
const context::notify_action ¬ify);
+
+ // Creates a clover module out of the given SPIR-V binary.
+ module process_program(const std::vector<char> &binary,
+ const device &dev, bool validate,
+ std::string &r_log);
+
+ // Combines multiple clover modules into a single one, resolving
+ // link dependencies between them.
+ module link_program(const std::vector<module> &modules, const device &dev,
+ const std::string &opts, std::string &r_log);
}
}
--
2.21.0
More information about the mesa-dev
mailing list