[Mesa-dev] [PATCH 1/3] glsl: delay optimisations on individual shaders when cache is available

Nicolai Hähnle nhaehnle at gmail.com
Wed Apr 5 10:00:19 UTC 2017


On 05.04.2017 05:56, Timothy Arceri wrote:
> Due to a max limit of 65,536 entries on the index table that we use to
> decide if we can skip compiling individual shaders, it is very likely
> we will have collisions.
>
> To avoid doing too much work when the linked program may be in the
> cache this patch delays calling the optimisations until link time.
>
> Improves cold cache start-up times on Deus Ex by ~20 seconds.
>
> When deleting the cache index to simulate a worst case scenario
> of collisions in the index, warm cache start-up time improves by
> ~45 seconds.
>
> V2: fix indentation, make sure to call optimisations on cache
> fallback, make sure optimisations get called for XFB.
>
> Tested-by: Grazvydas Ignotas <notasas at gmail.com>

Reviewed-by: Nicolai Hähnle <nicolai.haehnle at amd.com>


> ---
>  src/compiler/glsl/glsl_parser_extras.cpp | 166 +++++++++++++++++--------------
>  src/compiler/glsl/linker.cpp             |   3 -
>  src/compiler/glsl/shader_cache.cpp       |   2 +-
>  src/mesa/main/mtypes.h                   |   3 +-
>  4 files changed, 96 insertions(+), 78 deletions(-)
>
> diff --git a/src/compiler/glsl/glsl_parser_extras.cpp b/src/compiler/glsl/glsl_parser_extras.cpp
> index 8b5df3b..ec7860d0 100644
> --- a/src/compiler/glsl/glsl_parser_extras.cpp
> +++ b/src/compiler/glsl/glsl_parser_extras.cpp
> @@ -1914,20 +1914,99 @@ static void
>  do_late_parsing_checks(struct _mesa_glsl_parse_state *state)
>  {
>     if (state->stage == MESA_SHADER_COMPUTE && !state->has_compute_shader()) {
>        YYLTYPE loc;
>        memset(&loc, 0, sizeof(loc));
>        _mesa_glsl_error(&loc, state, "Compute shaders require "
>                         "GLSL 4.30 or GLSL ES 3.10");
>     }
>  }
>
> +static void
> +opt_shader_and_create_symbol_table(struct gl_context *ctx,
> +                                   struct gl_shader *shader)
> +{
> +   assert(shader->CompileStatus != compile_failure &&
> +          !shader->ir->is_empty());
> +
> +   struct gl_shader_compiler_options *options =
> +      &ctx->Const.ShaderCompilerOptions[shader->Stage];
> +
> +   /* Do some optimization at compile time to reduce shader IR size
> +    * and reduce later work if the same shader is linked multiple times
> +    */
> +   if (ctx->Const.GLSLOptimizeConservatively) {
> +      /* Run it just once. */
> +      do_common_optimization(shader->ir, false, false, options,
> +                             ctx->Const.NativeIntegers);
> +   } else {
> +      /* Repeat it until it stops making changes. */
> +      while (do_common_optimization(shader->ir, false, false, options,
> +                                    ctx->Const.NativeIntegers))
> +         ;
> +   }
> +
> +   validate_ir_tree(shader->ir);
> +
> +   enum ir_variable_mode other;
> +   switch (shader->Stage) {
> +   case MESA_SHADER_VERTEX:
> +      other = ir_var_shader_in;
> +      break;
> +   case MESA_SHADER_FRAGMENT:
> +      other = ir_var_shader_out;
> +      break;
> +   default:
> +      /* Something invalid to ensure optimize_dead_builtin_uniforms
> +       * doesn't remove anything other than uniforms or constants.
> +       */
> +      other = ir_var_mode_count;
> +      break;
> +   }
> +
> +   optimize_dead_builtin_variables(shader->ir, other);
> +
> +   validate_ir_tree(shader->ir);
> +
> +   /* Retain any live IR, but trash the rest. */
> +   reparent_ir(shader->ir, shader->ir);
> +
> +   /* Destroy the symbol table.  Create a new symbol table that contains only
> +    * the variables and functions that still exist in the IR.  The symbol
> +    * table will be used later during linking.
> +    *
> +    * There must NOT be any freed objects still referenced by the symbol
> +    * table.  That could cause the linker to dereference freed memory.
> +    *
> +    * We don't have to worry about types or interface-types here because those
> +    * are fly-weights that are looked up by glsl_type.
> +    */
> +   foreach_in_list (ir_instruction, ir, shader->ir) {
> +      switch (ir->ir_type) {
> +      case ir_type_function:
> +         shader->symbols->add_function((ir_function *) ir);
> +         break;
> +      case ir_type_variable: {
> +         ir_variable *const var = (ir_variable *) ir;
> +
> +         if (var->data.mode != ir_var_temporary)
> +            shader->symbols->add_variable(var);
> +         break;
> +      }
> +      default:
> +         break;
> +      }
> +   }
> +
> +   _mesa_glsl_initialize_derived_variables(ctx, shader);
> +}
> +
>  void
>  _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
>                            bool dump_ast, bool dump_hir, bool force_recompile)
>  {
>     struct _mesa_glsl_parse_state *state =
>        new(shader) _mesa_glsl_parse_state(ctx, shader->Stage, shader);
>     const char *source = force_recompile && shader->FallbackSource ?
>        shader->FallbackSource : shader->Source;
>
>     if (ctx->Const.GenerateTemporaryNames)
> @@ -1955,20 +2034,26 @@ _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
>              return;
>           }
>        }
>     } else {
>        /* We should only ever end up here if a re-compile has been forced by a
>         * shader cache miss. In which case we can skip the compile if its
>         * already be done by a previous fallback or the initial compile call.
>         */
>        if (shader->CompileStatus == compile_success)
>           return;
> +
> +      if (shader->CompileStatus == compiled_no_opts) {
> +         opt_shader_and_create_symbol_table(ctx, shader);
> +         shader->CompileStatus = compile_success;
> +         return;
> +      }
>     }
>
>     if (!state->error) {
>       _mesa_glsl_lexer_ctor(state, source);
>       _mesa_glsl_parse(state);
>       _mesa_glsl_lexer_dtor(state);
>       do_late_parsing_checks(state);
>     }
>
>     if (dump_ast) {
> @@ -1985,109 +2070,44 @@ _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
>
>     if (!state->error) {
>        validate_ir_tree(shader->ir);
>
>        /* Print out the unoptimized IR. */
>        if (dump_hir) {
>           _mesa_print_ir(stdout, shader->ir, state);
>        }
>     }
>
> -
> -   if (!state->error && !shader->ir->is_empty()) {
> -      struct gl_shader_compiler_options *options =
> -         &ctx->Const.ShaderCompilerOptions[shader->Stage];
> -
> -      assign_subroutine_indexes(shader, state);
> -      lower_subroutine(shader->ir, state);
> -
> -      /* Do some optimization at compile time to reduce shader IR size
> -       * and reduce later work if the same shader is linked multiple times
> -       */
> -      if (ctx->Const.GLSLOptimizeConservatively) {
> -         /* Run it just once. */
> -         do_common_optimization(shader->ir, false, false, options,
> -                                ctx->Const.NativeIntegers);
> -      } else {
> -         /* Repeat it until it stops making changes. */
> -         while (do_common_optimization(shader->ir, false, false, options,
> -                                       ctx->Const.NativeIntegers))
> -            ;
> -      }
> -
> -      validate_ir_tree(shader->ir);
> -
> -      enum ir_variable_mode other;
> -      switch (shader->Stage) {
> -      case MESA_SHADER_VERTEX:
> -         other = ir_var_shader_in;
> -         break;
> -      case MESA_SHADER_FRAGMENT:
> -         other = ir_var_shader_out;
> -         break;
> -      default:
> -         /* Something invalid to ensure optimize_dead_builtin_uniforms
> -          * doesn't remove anything other than uniforms or constants.
> -          */
> -         other = ir_var_mode_count;
> -         break;
> -      }
> -
> -      optimize_dead_builtin_variables(shader->ir, other);
> -
> -      validate_ir_tree(shader->ir);
> -   }
> -
>     if (shader->InfoLog)
>        ralloc_free(shader->InfoLog);
>
>     if (!state->error)
>        set_shader_inout_layout(shader, state);
>
>     shader->symbols = new(shader->ir) glsl_symbol_table;
>     shader->CompileStatus = state->error ? compile_failure : compile_success;
>     shader->InfoLog = state->info_log;
>     shader->Version = state->language_version;
>     shader->IsES = state->es_shader;
>
> -   /* Retain any live IR, but trash the rest. */
> -   reparent_ir(shader->ir, shader->ir);
> -
> -   /* Destroy the symbol table.  Create a new symbol table that contains only
> -    * the variables and functions that still exist in the IR.  The symbol
> -    * table will be used later during linking.
> -    *
> -    * There must NOT be any freed objects still referenced by the symbol
> -    * table.  That could cause the linker to dereference freed memory.
> -    *
> -    * We don't have to worry about types or interface-types here because those
> -    * are fly-weights that are looked up by glsl_type.
> -    */
> -   foreach_in_list (ir_instruction, ir, shader->ir) {
> -      switch (ir->ir_type) {
> -      case ir_type_function:
> -         shader->symbols->add_function((ir_function *) ir);
> -         break;
> -      case ir_type_variable: {
> -         ir_variable *const var = (ir_variable *) ir;
> +   if (!state->error && !shader->ir->is_empty()) {
> +      assign_subroutine_indexes(shader, state);
> +      lower_subroutine(shader->ir, state);
>
> -         if (var->data.mode != ir_var_temporary)
> -            shader->symbols->add_variable(var);
> -         break;
> -      }
> -      default:
> -         break;
> +      if (!ctx->Cache || force_recompile)
> +         opt_shader_and_create_symbol_table(ctx, shader);
> +      else {
> +         reparent_ir(shader->ir, shader->ir);
> +         shader->CompileStatus = compiled_no_opts;
>        }
>     }
>
> -   _mesa_glsl_initialize_derived_variables(ctx, shader);
> -
>     if (!force_recompile) {
>        free((void *)shader->FallbackSource);
>        shader->FallbackSource = NULL;
>     }
>
>     delete state->symbols;
>     ralloc_free(state);
>  }
>
>  } /* extern "C" */
> diff --git a/src/compiler/glsl/linker.cpp b/src/compiler/glsl/linker.cpp
> index 7bbffb7..7ace01d 100644
> --- a/src/compiler/glsl/linker.cpp
> +++ b/src/compiler/glsl/linker.cpp
> @@ -4634,23 +4634,20 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
>        if (ctx->API != API_OPENGL_COMPAT)
>           linker_error(prog, "no shaders attached to the program\n");
>        return;
>     }
>
>  #ifdef ENABLE_SHADER_CACHE
>     /* If transform feedback used on the program then compile all shaders. */
>     bool skip_cache = false;
>     if (prog->TransformFeedback.NumVarying > 0) {
>        for (unsigned i = 0; i < prog->NumShaders; i++) {
> -         if (prog->Shaders[i]->ir) {
> -            continue;
> -         }
>           _mesa_glsl_compile_shader(ctx, prog->Shaders[i], false, false, true);
>        }
>        skip_cache = true;
>     }
>
>     if (!skip_cache && shader_cache_read_program_metadata(ctx, prog))
>        return;
>  #endif
>
>     void *mem_ctx = ralloc_context(NULL); // temporary linker context
> diff --git a/src/compiler/glsl/shader_cache.cpp b/src/compiler/glsl/shader_cache.cpp
> index f5e6a22..e51fecd 100644
> --- a/src/compiler/glsl/shader_cache.cpp
> +++ b/src/compiler/glsl/shader_cache.cpp
> @@ -1411,21 +1411,21 @@ shader_cache_read_program_metadata(struct gl_context *ctx,
>     prog->data->LinkStatus = linking_skipped;
>
>     /* Since the program load was successful, CompileStatus of all shaders at
>      * this point should normally be compile_skipped. However because of how
>      * the eviction works, it may happen that some of the individual shader keys
>      * have been evicted, resulting in unnecessary recompiles on this load, so
>      * mark them again to skip such recompiles next time.
>      */
>     char sha1_buf[41];
>     for (unsigned i = 0; i < prog->NumShaders; i++) {
> -      if (prog->Shaders[i]->CompileStatus == compile_success) {
> +      if (prog->Shaders[i]->CompileStatus == compiled_no_opts) {
>           disk_cache_put_key(cache, prog->Shaders[i]->sha1);
>           if (ctx->_Shader->Flags & GLSL_CACHE_INFO) {
>              _mesa_sha1_format(sha1_buf, prog->Shaders[i]->sha1);
>              fprintf(stderr, "re-marking shader: %s\n", sha1_buf);
>           }
>        }
>     }
>
>     free (buffer);
>
> diff --git a/src/mesa/main/mtypes.h b/src/mesa/main/mtypes.h
> index 6fb6a51..1eedc8a 100644
> --- a/src/mesa/main/mtypes.h
> +++ b/src/mesa/main/mtypes.h
> @@ -2401,21 +2401,22 @@ static inline GLbitfield gl_external_samplers(struct gl_program *prog)
>
>  /**
>   * Compile status enum. compile_skipped is used to indicate the compile
>   * was skipped due to the shader matching one that's been seen before by
>   * the on-disk cache.
>   */
>  enum gl_compile_status
>  {
>     compile_failure = 0,
>     compile_success,
> -   compile_skipped
> +   compile_skipped,
> +   compiled_no_opts
>  };
>
>  /**
>   * A GLSL shader object.
>   */
>  struct gl_shader
>  {
>     /** GL_FRAGMENT_SHADER || GL_VERTEX_SHADER || GL_GEOMETRY_SHADER_ARB ||
>      *  GL_TESS_CONTROL_SHADER || GL_TESS_EVALUATION_SHADER.
>      * Must be the first field.
>


-- 
Lerne, wie die Welt wirklich ist,
Aber vergiss niemals, wie sie sein sollte.


More information about the mesa-dev mailing list