[Mesa-dev] [PATCH 11/12] spirv: Generate code to track SPIR-V capability dependencies

Eric Engestrom eric.engestrom at imgtec.com
Tue Nov 21 11:29:51 UTC 2017


On Monday, 2017-11-20 17:24:09 -0800, Ian Romanick wrote:
> From: Ian Romanick <ian.d.romanick at intel.com>
> 
> v2: Clean ups.  Remove some functions that never ended up being used.
> 
> v3: After updating spirv.core.grammar.json, fix the handling of
> ShaderViewportMaskNV.  See the comment around line 71 of
> spirv_capabilities_h.py.
> 
> v4: Many Python style changes based on feedback from Dylan.
> 
> Signed-off-by: Ian Romanick <ian.d.romanick at intel.com>
> ---
>  src/compiler/Makefile.sources              |   2 +
>  src/compiler/Makefile.spirv.am             |   8 +
>  src/compiler/glsl/meson.build              |   5 +-
>  src/compiler/spirv/.gitignore              |   2 +
>  src/compiler/spirv/meson.build             |  14 ++
>  src/compiler/spirv/spirv_capabilities_h.py | 365 +++++++++++++++++++++++++++++
>  6 files changed, 394 insertions(+), 2 deletions(-)
>  create mode 100644 src/compiler/spirv/spirv_capabilities_h.py
> 
> diff --git a/src/compiler/Makefile.sources b/src/compiler/Makefile.sources
> index 2ab8e16..1d67cba 100644
> --- a/src/compiler/Makefile.sources
> +++ b/src/compiler/Makefile.sources
> @@ -287,6 +287,8 @@ NIR_FILES = \
>  	nir/nir_worklist.h
>  
>  SPIRV_GENERATED_FILES = \
> +	spirv/spirv_capabilities.cpp \
> +	spirv/spirv_capabilities.h \
>  	spirv/spirv_info.c
>  
>  SPIRV_FILES = \
> diff --git a/src/compiler/Makefile.spirv.am b/src/compiler/Makefile.spirv.am
> index 9841004..4bc684a 100644
> --- a/src/compiler/Makefile.spirv.am
> +++ b/src/compiler/Makefile.spirv.am
> @@ -20,6 +20,14 @@
>  # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
>  # IN THE SOFTWARE.
>  
> +spirv/spirv_capabilities.cpp: spirv/spirv_capabilities_h.py spirv/spirv.core.grammar.json
> +	$(MKDIR_GEN)
> +	$(PYTHON_GEN) $(srcdir)/spirv/spirv_capabilities_h.py $(srcdir)/spirv/spirv.core.grammar.json $@ || ($(RM) $@; false)
> +
> +spirv/spirv_capabilities.h: spirv/spirv_capabilities_h.py spirv/spirv.core.grammar.json
> +	$(MKDIR_GEN)
> +	$(PYTHON_GEN) $(srcdir)/spirv/spirv_capabilities_h.py $(srcdir)/spirv/spirv.core.grammar.json $@ || ($(RM) $@; false)

Missing `--gen-cpp`/`--gen-h`

> +
>  spirv/spirv_info.c: spirv/spirv_info_c.py spirv/spirv.core.grammar.json
>  	$(MKDIR_GEN)
>  	$(PYTHON_GEN) $(srcdir)/spirv/spirv_info_c.py $(srcdir)/spirv/spirv.core.grammar.json $@ || ($(RM) $@; false)
> diff --git a/src/compiler/glsl/meson.build b/src/compiler/glsl/meson.build
> index 5b505c0..6e43f80 100644
> --- a/src/compiler/glsl/meson.build
> +++ b/src/compiler/glsl/meson.build
> @@ -198,11 +198,12 @@ files_libglsl_standalone = files(
>  libglsl = static_library(
>    'glsl',
>    [files_libglsl, glsl_parser, glsl_lexer_cpp, ir_expression_operation_h,
> -   ir_expression_operation_strings_h, ir_expression_operation_constant_h],
> +   ir_expression_operation_strings_h, ir_expression_operation_constant_h,
> +   spirv_capabilities_cpp, spirv_capabilities_h],
>    c_args : [c_vis_args, c_msvc_compat_args, no_override_init_args],
>    cpp_args : [cpp_vis_args, cpp_msvc_compat_args],
>    link_with : [libnir, libglcpp],
> -  include_directories : [inc_common, inc_compiler, inc_nir],
> +  include_directories : [inc_common, inc_compiler, inc_nir, inc_spirv],
>    build_by_default : false,
>  )
>  
> diff --git a/src/compiler/spirv/.gitignore b/src/compiler/spirv/.gitignore
> index f723c31..6b5ef0a 100644
> --- a/src/compiler/spirv/.gitignore
> +++ b/src/compiler/spirv/.gitignore
> @@ -1 +1,3 @@
> +/spirv_capabilities.cpp
> +/spirv_capabilities.h
>  /spirv_info.c
> diff --git a/src/compiler/spirv/meson.build b/src/compiler/spirv/meson.build
> index 8f1a02e..8b6071a 100644
> --- a/src/compiler/spirv/meson.build
> +++ b/src/compiler/spirv/meson.build
> @@ -24,3 +24,17 @@ spirv_info_c = custom_target(
>    output : 'spirv_info.c',
>    command : [prog_python2, '@INPUT0@', '@INPUT1@', '@OUTPUT@'],
>  )
> +
> +spirv_capabilities_cpp = custom_target(
> +  'spirv_capabilities.cpp',
> +  input : files('spirv_capabilities_h.py', 'spirv.core.grammar.json'),
> +  output : 'spirv_capabilities.cpp',
> +  command : [prog_python2, '@INPUT0@', '--gen-cpp', '@OUTPUT@', '@INPUT1@'],
> +)
> +
> +spirv_capabilities_h = custom_target(
> +  'spirv_capabilities.h',
> +  input : files('spirv_capabilities_h.py', 'spirv.core.grammar.json'),
> +  output : 'spirv_capabilities.h',
> +  command : [prog_python2, '@INPUT0@', '--gen-h', '@OUTPUT@', '@INPUT1@'],
> +)
> diff --git a/src/compiler/spirv/spirv_capabilities_h.py b/src/compiler/spirv/spirv_capabilities_h.py
> new file mode 100644
> index 0000000..78b1166
> --- /dev/null
> +++ b/src/compiler/spirv/spirv_capabilities_h.py
> @@ -0,0 +1,365 @@
> +COPYRIGHT = """\
> +/*
> + * Copyright (C) 2017 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> + * DEALINGS IN THE SOFTWARE.
> + */
> +"""
> +
> +import argparse
> +import json
> +from sys import stdout, stderr, exit
> +from mako.template import Template
> +
> +def parse_args():
> +    p = argparse.ArgumentParser()
> +    p.add_argument("json", help="SPIR-V definition JSON")
> +    p.add_argument("--gen-cpp", help="Name of the C++ file to generate")
> +    p.add_argument("--gen-h", help="Name of the header file to generate")
> +    return p.parse_args()
> +
> +def collect_data(spirv):
> +    for x in spirv["operand_kinds"]:
> +        # There should only be one set of Capability enums in the JSON.
> +        if x["kind"] == "Capability":
> +            operands = x
> +            break
> +
> +    # There are some duplicate values in some of the tables (thanks guys!), so
> +    # filter them out.
> +
> +    # by_value is a dictionary that maps values of enumerants to tuples of
> +    # enumerant names and capabilities.
> +    by_value = {}
> +
> +    # by_name is a dictionary that maps names of enumerants to tuples of
> +    # values and required capabilities.
> +    by_name = {}
> +
> +    for x in operands["enumerants"]:
> +        caps = x.get("capabilities", [])
> +
> +        if x["value"] not in by_value:
> +            by_value[x["value"]] = (x["enumerant"], caps)
> +
> +        by_name[x["enumerant"]] = (x["value"], caps)
> +
> +    # Recall that there are some duplicate values in the table.  These
> +    # duplicate values also appear in the "capabilities" list for some
> +    # enumerants.  Filter out the duplicates there as well.
> +    for capability, (_, dependencies) in by_name.iteritems():
> +        for dependency in dependencies:
> +            dep_value, _ = by_name[dependency]
> +            real_dependency, _ = by_value[dep_value]
> +
> +            # In most cases where there is a duplicate capability, things that
> +            # depend on one will also depend on the others.
> +            # StorageBuffer16BitAccess and StorageUniformBufferBlock16 have
> +            # the same value, and StorageUniform16 depends on both.
> +            #
> +            # There are exceptions.  ShaderViewportIndexLayerEXT and
> +            # ShaderViewportIndexLayerNV have the same value, but
> +            # ShaderViewportMaskNV only depends on ShaderViewportIndexLayerNV.
> +            #
> +            # That's the only case so far, so emit a warning for other cases
> +            # that have more than one dependency.  That way we can double
> +            # check that they are handled correctly.
> +
> +            if real_dependency != dependency:
> +                if real_dependency not in by_name[capability][1]:
> +                    if len(by_name[capability][1]) > 1:
> +                        print("Warning!  Removed {} from {}, but no name with the same value is in the dependency list.".format(dependency, capability))
> +                else:
> +                    if len(by_name[capability][1]) == 1:
> +                        stderr.write("Error!  Cannot remove {} from {} because it is the only dependency.\n".format(dependency, capability))
> +                        exit(1)
> +
> +                    by_name[capability][1].remove(dependency)
> +
> +        # The table generated from this data and the C code that uses it
> +        # assumes that each capability has a single dependency.  That is
> +        # currently the case, but it may change in the future.
> +        if len(by_name[capability][1]) > 1:
> +            stderr.write("Error!  Too many dependencies left for {}. {}\n".format(capability, by_name[capability][1]))
> +            exit(1)
> +
> +    for cap_value in by_value:
> +        name, _ = by_value[cap_value]
> +        by_value[cap_value] = (name, by_name[name][1])
> +
> +    return (by_name, by_value)
> +
> +TEMPLATE_H = Template(COPYRIGHT + """\
> +#ifndef SPIRV_CAPABILITIES_H
> +#define SPIRV_CAPABILITIES_H
> +
> +#include <stdint.h>
> +#include "spirv.h"
> +#include "util/bitset.h"
> +#ifndef ARRAY_SIZE
> +#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
> +#endif
> +
> +#define NO_DEPENDENCY ((${"uint16_t" if len(all_values) > 256 else "uint8_t"}) ~0)
> +
> +class spirv_capability_set;
> +
> +/**
> + * Iterator for the enabled capabilities in a spirv_capability_set
> + *
> + * Roughly, this is a wrapper for the bitset iterator functions.  Dereferencing
> + * the iterator results in the \c SpvCapability where as the bitset iterator
> + * functions provide the index in the bitset.  The mapping is handled by
> + * \c spirv_capability_set.
> + */
> +class spirv_capability_set_iterator {
> +public:
> +   spirv_capability_set_iterator(const spirv_capability_set *_s);
> +   inline bool operator==(const spirv_capability_set_iterator &that) const;
> +   inline bool operator!=(const spirv_capability_set_iterator &that) const;
> +   spirv_capability_set_iterator &operator++();
> +   SpvCapability operator*() const;
> +
> +private:
> +   /** Capability set being iterated. */
> +   const spirv_capability_set *s;
> +
> +   /** Current index in the set. */
> +   unsigned i;
> +
> +   /** Temporary storage used by \c __bitset_next_set */
> +   BITSET_WORD tmp;
> +
> +   /**
> +    * Construct an iterator starting a specific index
> +    *
> +    * Used by \c spirv_capability_set::end().
> +    */
> +   spirv_capability_set_iterator(const spirv_capability_set *_s, unsigned _i);
> +
> +   friend class spirv_capability_set;
> +};
> +
> +/**
> + * Set of SPIR-V capabilities
> + *
> + * All SPIR-V capability values are mapped to a compacted bitset.  The mapping
> + * of implemented in the (private) methods \c ::capability_as_index() and
> + * \c ::index_as_capability().  Once all of the capabilities for a particular
> + * SPIR-V program have been enabled (by repeatedly calling \c ::enable()), the
> + * set can be reduced to the minimum set by \c ::reduce().
> + */
> +class spirv_capability_set {
> +public:
> +   /**
> +    * Construct a capability set with no capabilities enabled.
> +    */
> +   spirv_capability_set()
> +   {
> +      BITSET_ZERO(capabilities);
> +   }
> +
> +   /**
> +    * Enable the specified SPIR-V capability
> +    */
> +   inline void enable(SpvCapability cap)
> +   {
> +      BITSET_SET(capabilities, capability_as_index(cap));
> +   }
> +
> +   /**
> +    * Determine whether or not the specified SPIR-V capability is enabled
> +    */
> +   inline bool is_enabled(SpvCapability cap)
> +   {
> +      return BITSET_TEST(capabilities, capability_as_index(cap));
> +   }
> +
> +   /**
> +    * Reduce a set of SPIR-V capabilities to the minimal set.
> +    *
> +    * Many SPIR-V capabilities imply other capabilities.  For example,
> +    * \c SpvCapabilityShader implies \c SpvCapabilityMatrix, so only the
> +    * former needs to be set.  This method removes all the redundant
> +    * capabilities so that the minimal set of \c SpvOpCapability instructions
> +    * can be written to the output.
> +    */
> +   void reduce()
> +   {
> +      for (unsigned i = 0; i < ${len(all_values)}; ++i) {
> +         if (BITSET_TEST(capabilities, i)) {
> +            /* Walk back up the dependency chain clearing all the bits along
> +             * the way.  This is necessary because some of the dependencies
> +             * might not be set, so we cannot rely on the dependency-of-a-
> +             * dependency to be cleared automatically.
> +             */
> +            unsigned dep = dependency_map[i];
> +            while (dep != NO_DEPENDENCY) {
> +                BITSET_CLEAR(capabilities, dep);
> +                dep = dependency_map[dep];
> +            }
> +         }
> +      }
> +   }
> +
> +   spirv_capability_set_iterator begin() const
> +   {
> +      return spirv_capability_set_iterator(this);
> +   }
> +
> +   spirv_capability_set_iterator end() const
> +   {
> +      return spirv_capability_set_iterator(this, ${len(all_values)});
> +   }
> +
> +private:
> +   /**
> +    * Map a SPIR-V capability to its location in the bitfield.
> +    */
> +   static unsigned capability_as_index(SpvCapability cap)
> +   {
> +      if (cap <= SpvCapability${by_value[[x for x in all_values if x < 4096][-1]][0]})
> +         return unsigned(cap);
> +
> +      switch (cap) {
> +    % for x in all_values:
> +        % if x >= 4096:
> +      case SpvCapability${by_value[x][0]}: return ${all_values.index(x)};
> +        % endif
> +    % endfor
> +      default:
> +         unreachable("Invalid capability.");
> +      }
> +   }
> +
> +   /**
> +    * Map a location in the bitfield to the SPIR-V capability it represents.
> +    */
> +   static SpvCapability index_as_capability(unsigned i)
> +   {
> +      if (i <= ${[x for x in all_values if x < 4096][-1]})
> +         return SpvCapability(i);
> +
> +      switch (i) {
> +    % for x in all_values:
> +        % if x >= 4096:
> +      case ${all_values.index(x)}: return SpvCapability${by_value[x][0]};
> +        % endif
> +    % endfor
> +      default:
> +         unreachable("Invalid capability index.");
> +      }
> +   }
> +
> +   BITSET_DECLARE(capabilities, ${len(all_values)});
> +
> +   static const ${"uint16_t" if len(all_values) > 256 else "uint8_t"} dependency_map[${len(all_values)}];
> +
> +   friend class spirv_capability_set_iterator;
> +};
> +
> +inline spirv_capability_set_iterator::spirv_capability_set_iterator(const spirv_capability_set *_s)
> +   : s(_s), i(0), tmp(_s->capabilities[0])
> +{
> +   i = __bitset_next_set(i, &tmp, s->capabilities, ${len(all_values)});
> +}
> +
> +inline spirv_capability_set_iterator::spirv_capability_set_iterator(
> +   const spirv_capability_set *_s, unsigned _i)
> +   : s(_s), i(_i)
> +{
> +   tmp = s->capabilities[unsigned(i / BITSET_WORDBITS)];
> +   tmp &= ~((1u << (i % BITSET_WORDBITS))- 1);
> +}
> +
> +inline bool spirv_capability_set_iterator::operator==(const spirv_capability_set_iterator &that) const
> +{
> +   return s == that.s && i == that.i;
> +}
> +
> +inline bool spirv_capability_set_iterator::operator!=(const spirv_capability_set_iterator &that) const
> +{
> +   return !(*this == that);
> +}
> +
> +inline spirv_capability_set_iterator &spirv_capability_set_iterator::operator++()
> +{
> +   i = __bitset_next_set(i, &tmp, s->capabilities, ${len(all_values)});
> +   return *this;
> +}
> +
> +inline SpvCapability spirv_capability_set_iterator::operator*() const
> +{
> +   return spirv_capability_set::index_as_capability(i);
> +}
> +
> +#undef NO_DEPENDENCY
> +
> +#endif /* SPIRV_CAPABILITIES_H */
> +""")
> +
> +
> +TEMPLATE_CPP = Template("""\
> +/* DO NOT EDIT - This file is generated automatically by spirv_capabilities_h.py script */
> +
> +""" + COPYRIGHT + """\
> +#include "util/macros.h" /* for unreachable() */
> +#include "spirv_capabilities.h"
> +
> +#define NO_DEPENDENCY ((${"uint16_t" if len(all_values) > 256 else "uint8_t"}) ~0)
> +
> +const ${"uint16_t" if len(all_values) > 256 else "uint8_t"} spirv_capability_set::dependency_map[${len(all_values)}] = {
> +    % for x in all_values:
> +        % if x not in by_value or len(by_value[x][1]) == 0:
> +   NO_DEPENDENCY${"," if all_values[-1] != x else ""}
> +        % else:
> +   ${all_values.index(by_name[by_value[x][1][0]][0])}${"," if all_values[-1] != x else ""}
> +        % endif
> +    % endfor
> +};
> +""")
> +
> +if __name__ == "__main__":
> +    pargs = parse_args()
> +
> +    if not pargs.gen_cpp and not pargs.gen_h:
> +        stderr.write("At least one of --gen-cpp or --gen-h must be supplied.\n")
> +        exit(1)
> +
> +    with open(pargs.json, 'r') as f:
> +        spirv_info = json.load(f)
> +
> +    by_name, by_value = collect_data(spirv_info)
> +
> +    # Assume the "core" values will be fairly tightly packed.
> +    max_core_value = max([x for _, (x, _) in by_name.items() if int(x) < 4096])
> +    all_values = [x for x in xrange(max_core_value + 1)] + [cap_value for cap_value, _ in sorted(by_value.items()) if cap_value >= 4096]
> +
> +    if pargs.gen_cpp:
> +        with open(pargs.gen_cpp, 'w') as f:
> +            f.write(TEMPLATE_CPP.render(by_name=by_name,
> +                                        by_value=by_value,
> +                                        all_values=all_values))
> +
> +    if pargs.gen_h:
> +        with open(pargs.gen_h, 'w') as f:
> +            f.write(TEMPLATE_H.render(by_name=by_name,
> +                                      by_value=by_value,
> +                                      all_values=all_values))
> -- 
> 2.9.5
> 


More information about the mesa-dev mailing list