[Mesa-dev] [PATCH 1/3] glsl: delay optimisations on individual shaders when cache is available
Nicolai Hähnle
nhaehnle at gmail.com
Wed Apr 5 10:00:19 UTC 2017
On 05.04.2017 05:56, Timothy Arceri wrote:
> Due to a max limit of 65,536 entries on the index table that we use to
> decide if we can skip compiling individual shaders, it is very likely
> we will have collisions.
>
> To avoid doing too much work when the linked program may be in the
> cache this patch delays calling the optimisations until link time.
>
> Improves cold cache start-up times on Deus Ex by ~20 seconds.
>
> When deleting the cache index to simulate a worst case scenario
> of collisions in the index, warm cache start-up time improves by
> ~45 seconds.
>
> V2: fix indentation, make sure to call optimisations on cache
> fallback, make sure optimisations get called for XFB.
>
> Tested-by: Grazvydas Ignotas <notasas at gmail.com>
Reviewed-by: Nicolai Hähnle <nicolai.haehnle at amd.com>
> ---
> src/compiler/glsl/glsl_parser_extras.cpp | 166 +++++++++++++++++--------------
> src/compiler/glsl/linker.cpp | 3 -
> src/compiler/glsl/shader_cache.cpp | 2 +-
> src/mesa/main/mtypes.h | 3 +-
> 4 files changed, 96 insertions(+), 78 deletions(-)
>
> diff --git a/src/compiler/glsl/glsl_parser_extras.cpp b/src/compiler/glsl/glsl_parser_extras.cpp
> index 8b5df3b..ec7860d0 100644
> --- a/src/compiler/glsl/glsl_parser_extras.cpp
> +++ b/src/compiler/glsl/glsl_parser_extras.cpp
> @@ -1914,20 +1914,99 @@ static void
> do_late_parsing_checks(struct _mesa_glsl_parse_state *state)
> {
> if (state->stage == MESA_SHADER_COMPUTE && !state->has_compute_shader()) {
> YYLTYPE loc;
> memset(&loc, 0, sizeof(loc));
> _mesa_glsl_error(&loc, state, "Compute shaders require "
> "GLSL 4.30 or GLSL ES 3.10");
> }
> }
>
> +static void
> +opt_shader_and_create_symbol_table(struct gl_context *ctx,
> + struct gl_shader *shader)
> +{
> + assert(shader->CompileStatus != compile_failure &&
> + !shader->ir->is_empty());
> +
> + struct gl_shader_compiler_options *options =
> + &ctx->Const.ShaderCompilerOptions[shader->Stage];
> +
> + /* Do some optimization at compile time to reduce shader IR size
> + * and reduce later work if the same shader is linked multiple times
> + */
> + if (ctx->Const.GLSLOptimizeConservatively) {
> + /* Run it just once. */
> + do_common_optimization(shader->ir, false, false, options,
> + ctx->Const.NativeIntegers);
> + } else {
> + /* Repeat it until it stops making changes. */
> + while (do_common_optimization(shader->ir, false, false, options,
> + ctx->Const.NativeIntegers))
> + ;
> + }
> +
> + validate_ir_tree(shader->ir);
> +
> + enum ir_variable_mode other;
> + switch (shader->Stage) {
> + case MESA_SHADER_VERTEX:
> + other = ir_var_shader_in;
> + break;
> + case MESA_SHADER_FRAGMENT:
> + other = ir_var_shader_out;
> + break;
> + default:
> + /* Something invalid to ensure optimize_dead_builtin_uniforms
> + * doesn't remove anything other than uniforms or constants.
> + */
> + other = ir_var_mode_count;
> + break;
> + }
> +
> + optimize_dead_builtin_variables(shader->ir, other);
> +
> + validate_ir_tree(shader->ir);
> +
> + /* Retain any live IR, but trash the rest. */
> + reparent_ir(shader->ir, shader->ir);
> +
> + /* Destroy the symbol table. Create a new symbol table that contains only
> + * the variables and functions that still exist in the IR. The symbol
> + * table will be used later during linking.
> + *
> + * There must NOT be any freed objects still referenced by the symbol
> + * table. That could cause the linker to dereference freed memory.
> + *
> + * We don't have to worry about types or interface-types here because those
> + * are fly-weights that are looked up by glsl_type.
> + */
> + foreach_in_list (ir_instruction, ir, shader->ir) {
> + switch (ir->ir_type) {
> + case ir_type_function:
> + shader->symbols->add_function((ir_function *) ir);
> + break;
> + case ir_type_variable: {
> + ir_variable *const var = (ir_variable *) ir;
> +
> + if (var->data.mode != ir_var_temporary)
> + shader->symbols->add_variable(var);
> + break;
> + }
> + default:
> + break;
> + }
> + }
> +
> + _mesa_glsl_initialize_derived_variables(ctx, shader);
> +}
> +
> void
> _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
> bool dump_ast, bool dump_hir, bool force_recompile)
> {
> struct _mesa_glsl_parse_state *state =
> new(shader) _mesa_glsl_parse_state(ctx, shader->Stage, shader);
> const char *source = force_recompile && shader->FallbackSource ?
> shader->FallbackSource : shader->Source;
>
> if (ctx->Const.GenerateTemporaryNames)
> @@ -1955,20 +2034,26 @@ _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
> return;
> }
> }
> } else {
> /* We should only ever end up here if a re-compile has been forced by a
> * shader cache miss. In which case we can skip the compile if its
> * already be done by a previous fallback or the initial compile call.
> */
> if (shader->CompileStatus == compile_success)
> return;
> +
> + if (shader->CompileStatus == compiled_no_opts) {
> + opt_shader_and_create_symbol_table(ctx, shader);
> + shader->CompileStatus = compile_success;
> + return;
> + }
> }
>
> if (!state->error) {
> _mesa_glsl_lexer_ctor(state, source);
> _mesa_glsl_parse(state);
> _mesa_glsl_lexer_dtor(state);
> do_late_parsing_checks(state);
> }
>
> if (dump_ast) {
> @@ -1985,109 +2070,44 @@ _mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader,
>
> if (!state->error) {
> validate_ir_tree(shader->ir);
>
> /* Print out the unoptimized IR. */
> if (dump_hir) {
> _mesa_print_ir(stdout, shader->ir, state);
> }
> }
>
> -
> - if (!state->error && !shader->ir->is_empty()) {
> - struct gl_shader_compiler_options *options =
> - &ctx->Const.ShaderCompilerOptions[shader->Stage];
> -
> - assign_subroutine_indexes(shader, state);
> - lower_subroutine(shader->ir, state);
> -
> - /* Do some optimization at compile time to reduce shader IR size
> - * and reduce later work if the same shader is linked multiple times
> - */
> - if (ctx->Const.GLSLOptimizeConservatively) {
> - /* Run it just once. */
> - do_common_optimization(shader->ir, false, false, options,
> - ctx->Const.NativeIntegers);
> - } else {
> - /* Repeat it until it stops making changes. */
> - while (do_common_optimization(shader->ir, false, false, options,
> - ctx->Const.NativeIntegers))
> - ;
> - }
> -
> - validate_ir_tree(shader->ir);
> -
> - enum ir_variable_mode other;
> - switch (shader->Stage) {
> - case MESA_SHADER_VERTEX:
> - other = ir_var_shader_in;
> - break;
> - case MESA_SHADER_FRAGMENT:
> - other = ir_var_shader_out;
> - break;
> - default:
> - /* Something invalid to ensure optimize_dead_builtin_uniforms
> - * doesn't remove anything other than uniforms or constants.
> - */
> - other = ir_var_mode_count;
> - break;
> - }
> -
> - optimize_dead_builtin_variables(shader->ir, other);
> -
> - validate_ir_tree(shader->ir);
> - }
> -
> if (shader->InfoLog)
> ralloc_free(shader->InfoLog);
>
> if (!state->error)
> set_shader_inout_layout(shader, state);
>
> shader->symbols = new(shader->ir) glsl_symbol_table;
> shader->CompileStatus = state->error ? compile_failure : compile_success;
> shader->InfoLog = state->info_log;
> shader->Version = state->language_version;
> shader->IsES = state->es_shader;
>
> - /* Retain any live IR, but trash the rest. */
> - reparent_ir(shader->ir, shader->ir);
> -
> - /* Destroy the symbol table. Create a new symbol table that contains only
> - * the variables and functions that still exist in the IR. The symbol
> - * table will be used later during linking.
> - *
> - * There must NOT be any freed objects still referenced by the symbol
> - * table. That could cause the linker to dereference freed memory.
> - *
> - * We don't have to worry about types or interface-types here because those
> - * are fly-weights that are looked up by glsl_type.
> - */
> - foreach_in_list (ir_instruction, ir, shader->ir) {
> - switch (ir->ir_type) {
> - case ir_type_function:
> - shader->symbols->add_function((ir_function *) ir);
> - break;
> - case ir_type_variable: {
> - ir_variable *const var = (ir_variable *) ir;
> + if (!state->error && !shader->ir->is_empty()) {
> + assign_subroutine_indexes(shader, state);
> + lower_subroutine(shader->ir, state);
>
> - if (var->data.mode != ir_var_temporary)
> - shader->symbols->add_variable(var);
> - break;
> - }
> - default:
> - break;
> + if (!ctx->Cache || force_recompile)
> + opt_shader_and_create_symbol_table(ctx, shader);
> + else {
> + reparent_ir(shader->ir, shader->ir);
> + shader->CompileStatus = compiled_no_opts;
> }
> }
>
> - _mesa_glsl_initialize_derived_variables(ctx, shader);
> -
> if (!force_recompile) {
> free((void *)shader->FallbackSource);
> shader->FallbackSource = NULL;
> }
>
> delete state->symbols;
> ralloc_free(state);
> }
>
> } /* extern "C" */
> diff --git a/src/compiler/glsl/linker.cpp b/src/compiler/glsl/linker.cpp
> index 7bbffb7..7ace01d 100644
> --- a/src/compiler/glsl/linker.cpp
> +++ b/src/compiler/glsl/linker.cpp
> @@ -4634,23 +4634,20 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
> if (ctx->API != API_OPENGL_COMPAT)
> linker_error(prog, "no shaders attached to the program\n");
> return;
> }
>
> #ifdef ENABLE_SHADER_CACHE
> /* If transform feedback used on the program then compile all shaders. */
> bool skip_cache = false;
> if (prog->TransformFeedback.NumVarying > 0) {
> for (unsigned i = 0; i < prog->NumShaders; i++) {
> - if (prog->Shaders[i]->ir) {
> - continue;
> - }
> _mesa_glsl_compile_shader(ctx, prog->Shaders[i], false, false, true);
> }
> skip_cache = true;
> }
>
> if (!skip_cache && shader_cache_read_program_metadata(ctx, prog))
> return;
> #endif
>
> void *mem_ctx = ralloc_context(NULL); // temporary linker context
> diff --git a/src/compiler/glsl/shader_cache.cpp b/src/compiler/glsl/shader_cache.cpp
> index f5e6a22..e51fecd 100644
> --- a/src/compiler/glsl/shader_cache.cpp
> +++ b/src/compiler/glsl/shader_cache.cpp
> @@ -1411,21 +1411,21 @@ shader_cache_read_program_metadata(struct gl_context *ctx,
> prog->data->LinkStatus = linking_skipped;
>
> /* Since the program load was successful, CompileStatus of all shaders at
> * this point should normally be compile_skipped. However because of how
> * the eviction works, it may happen that some of the individual shader keys
> * have been evicted, resulting in unnecessary recompiles on this load, so
> * mark them again to skip such recompiles next time.
> */
> char sha1_buf[41];
> for (unsigned i = 0; i < prog->NumShaders; i++) {
> - if (prog->Shaders[i]->CompileStatus == compile_success) {
> + if (prog->Shaders[i]->CompileStatus == compiled_no_opts) {
> disk_cache_put_key(cache, prog->Shaders[i]->sha1);
> if (ctx->_Shader->Flags & GLSL_CACHE_INFO) {
> _mesa_sha1_format(sha1_buf, prog->Shaders[i]->sha1);
> fprintf(stderr, "re-marking shader: %s\n", sha1_buf);
> }
> }
> }
>
> free (buffer);
>
> diff --git a/src/mesa/main/mtypes.h b/src/mesa/main/mtypes.h
> index 6fb6a51..1eedc8a 100644
> --- a/src/mesa/main/mtypes.h
> +++ b/src/mesa/main/mtypes.h
> @@ -2401,21 +2401,22 @@ static inline GLbitfield gl_external_samplers(struct gl_program *prog)
>
> /**
> * Compile status enum. compile_skipped is used to indicate the compile
> * was skipped due to the shader matching one that's been seen before by
> * the on-disk cache.
> */
> enum gl_compile_status
> {
> compile_failure = 0,
> compile_success,
> - compile_skipped
> + compile_skipped,
> + compiled_no_opts
> };
>
> /**
> * A GLSL shader object.
> */
> struct gl_shader
> {
> /** GL_FRAGMENT_SHADER || GL_VERTEX_SHADER || GL_GEOMETRY_SHADER_ARB ||
> * GL_TESS_CONTROL_SHADER || GL_TESS_EVALUATION_SHADER.
> * Must be the first field.
>
--
Lerne, wie die Welt wirklich ist,
Aber vergiss niemals, wie sie sein sollte.
More information about the mesa-dev
mailing list