<div dir="ltr">On 1 November 2013 02:16, Tapani Pälli <<a href="mailto:tapani.palli@intel.com" target="_blank">tapani.palli@intel.com</a>> wrote: <div class="gmail_extra"><div class="gmail_quote"> <blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">These utility functions can be used to (de)serialize gl_shader and gl_shader_program structures. This makes it possible to implement a shader compiler cache for individual shaders and functionality required by OES_get_program_binary extension. Signed-off-by: Tapani Pälli <<a href="mailto:tapani.palli@intel.com">tapani.palli@intel.com</a>> --- src/glsl/Makefile.sources | 1 + src/glsl/ir_cache.cpp | 373 ++++++++++++++++++++++++++++++++++++++++++++++ src/glsl/ir_cache.h | 58 +++++++ 3 files changed, 432 insertions(+) create mode 100644 src/glsl/ir_cache.cpp create mode 100644 src/glsl/ir_cache.h diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources index 81d5753..99b3c1a 100644 --- a/src/glsl/Makefile.sources +++ b/src/glsl/Makefile.sources @@ -30,6 +30,7 @@ LIBGLSL_FILES = \ $(GLSL_SRCDIR)/hir_field_selection.cpp \ $(GLSL_SRCDIR)/ir_basic_block.cpp \ $(GLSL_SRCDIR)/ir_builder.cpp \ + $(GLSL_SRCDIR)/ir_cache.cpp \ $(GLSL_SRCDIR)/ir_cache_serializer.cpp \ $(GLSL_SRCDIR)/ir_cache_deserializer.cpp \ $(GLSL_SRCDIR)/ir_clone.cpp \ diff --git a/src/glsl/ir_cache.cpp b/src/glsl/ir_cache.cpp new file mode 100644 index 0000000..24e1c77 --- /dev/null +++ b/src/glsl/ir_cache.cpp @@ -0,0 +1,373 @@ +/* -*- c++ -*- */ +/* + * Copyright © 2013 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "main/shaderobj.h" +#include "main/uniforms.h" +#include "main/macros.h" + +#include "ir_cache_serializer.h" +#include "ir_cache_deserializer.h" + +/** + * Serialize gl_shader structure + */ +extern "C" char * +_mesa_shader_serialize(struct gl_shader *shader, + struct _mesa_glsl_parse_state *state, + const char *mesa_sha, size_t *size) +{ + ir_serializer s; + return s.serialize(shader, state, mesa_sha, size); +} + + +static void +calc_item(const void *key, void *data, void *closure) </blockquote><div> </div><div>How about "increment_count" for the name of this function? </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +{ + unsigned *sz = (unsigned *) closure; + *sz = *sz + 1; +} + + +static unsigned +_hash_table_size(struct string_to_uint_map *map) </blockquote><div> </div><div>This function is global, so its name can't start with an underscore--such names are reserved for library functions. Same goes for many of the other functions in this file. </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +{ + unsigned size = 0; + map->iterate(calc_item, &size); + return size; +} + + +static void +serialize_item(const void *key, void *data, void *closure) +{ + memory_writer *blob = (memory_writer *) closure; + uint32_t value = ((intptr_t)data); + + blob->write_string((char *)key); + blob->write_uint32(&value); +} + + +static void +_serialize_hash_table(struct string_to_uint_map *map, memory_writer *blob) +{ + uint32_t size = _hash_table_size(map); + blob->write_uint32(&size); + map->iterate(serialize_item, blob); </blockquote><div> </div><div>Rather than take two passes over the hash table (one to compute its size and one to output its contents), why not do the trick that we do with overwrite() in ir_cache_serializer.cpp? (In other words, reserve space for the size of the hashtable in bytes, then serialize the hashtable, then overwrite the reserved space with the actual size). </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +} + + +static void +_serialize_uniform_storage(gl_uniform_storage *uni, memory_writer &blob) +{ + blob.write_string(uni->name); + + save_glsl_type(blob, uni->type); + + uint8_t initialized = uni->initialized; + uint8_t row_major = uni->row_major; + + blob.write_uint32(&uni->array_elements); + blob.write_uint8(&initialized); + blob.write_int32(&uni->block_index); + blob.write_int32(&uni->offset); + blob.write_int32(&uni->matrix_stride); + blob.write_uint8(&row_major); + blob.write_int32(&uni->atomic_buffer_index); + + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) { + uint8_t active = uni->sampler[i].active; + blob.write_uint8(&uni->sampler[i].index); + blob.write_uint8(&active); + } + + const unsigned elements = MAX2(1, uni->array_elements); + const unsigned data_components = elements * uni->type->components(); + uint32_t size = elements * MAX2(1, data_components); + + CACHE_DEBUG("%s: size %ld\n", __func__, + size * sizeof(union gl_constant_value)); + + blob.write_uint32(&size); + blob.write(uni->storage, size * sizeof(union gl_constant_value)); +} + + +/** + * Serialize gl_shader_program structure + */ +extern "C" char * +_mesa_program_serialize(struct gl_shader_program *prog, size_t *size, + const char *mesa_sha) +{ + memory_writer blob; + + blob.write_uint32(&prog->Type); + blob.write_uint32(&prog->NumShaders); + blob.write_uint8(&prog->LinkStatus); + blob.write_uint32(&prog->Version); + blob.write_uint8(&prog->IsES); + blob.write_uint32(&prog->NumUserUniformStorage); + blob.write_uint32(&prog->UniformLocationBaseScale); + + /* hash tables */ + _serialize_hash_table(prog->AttributeBindings, &blob); + _serialize_hash_table(prog->FragDataBindings, &blob); + _serialize_hash_table(prog->FragDataIndexBindings, &blob); + _serialize_hash_table(prog->UniformHash, &blob); + + /* uniform storage */ + if (prog->UniformStorage) { + for (unsigned i = 0; i < prog->NumUserUniformStorage; ++i) + _serialize_uniform_storage(&prog->UniformStorage[i], blob); + } + + /* Shaders IR, to be decided if we want these to be available */ </blockquote><div> </div><div>Has there been previous discussion on the mesa-dev list about whether or not we want these to be available? If so, please include a link to previous discussion and a brief summary. if not, what does the decision hinge on? At first blush it seems to me that there's no reason to serialize the IR of the individual shaders, since ARB_get_program_binary operates on full programs rather than individual shaders. </div><div>Also, typically we prefer to disable code using "if (false)" rather than ifdefs, because that ensures that the disabled code still compiles. (Otherwise, it's much more likely to bit rot as the rest of the code base evolves). </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +#if 0 + for (unsigned i = 0; i < prog->NumShaders; i++) { + size_t sha_size = 0; + char *data = _mesa_shader_serialize(prog->Shaders[i], + NULL, mesa_sha, &sha_size); + + if (data) { + blob.write(data, sha_size); + free(data); + } + } +#endif + + /* _LinkedShaders IR */ + for (uint32_t i = 0; i < MESA_SHADER_TYPES; i++) { + size_t sha_size = 0; + + if (!prog->_LinkedShaders[i]) + continue; + + char *data = _mesa_shader_serialize(prog->_LinkedShaders[i], + NULL, mesa_sha, &sha_size); + + if (!data) { + CACHE_DEBUG("error serializing data for index %d\n", i); + return NULL; + } + + /* index in _LinkedShaders list + shader blob */ + if (data) { + blob.write_uint32(&i); + blob.write(data, sha_size); + free(data); + } + } </blockquote><div> </div><div>Are we going to attempt to store the back-end compiled shaders? It seems like we'd have to do that in order to get a lot of the benefit of ARB_get_program_binary. </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> + + *size = blob.position(); + return blob.release_memory(size); +} + + +/** + * Deserialize gl_shader structure + */ +extern "C" struct gl_shader * +_mesa_shader_deserialize(void *mem_ctx, void *blob, + const char *mesa_sha, size_t size) +{ + int error = 0; + ir_deserializer s; + memory_map map; + + map.map(blob, size); + + return s.deserialize(mem_ctx, map, size, + NULL, + mesa_sha, + &error); +} + + +static void +_read_hash_table(struct string_to_uint_map *hash, memory_map *map) +{ + unsigned size; + map->read(&size); + + for (unsigned i = 0; i < size; i++) { + unsigned value; + + char *key = map->read_string(); + map->read(&value); + + hash->put(value-1, key); + } +} + + +static void +_read_uniform_storage(void *mem_ctx, gl_uniform_storage *uni, + memory_map &map, struct _mesa_glsl_parse_state *state) +{ + ir_deserializer s; + + char *name = map.read_string(); + uni->name = strdup(name); + + uni->type = s.read_glsl_type(map, NULL); + + map.read(&uni->array_elements); + map.read(&uni->initialized); + map.read(&uni->block_index); + map.read(&uni->offset); + map.read(&uni->matrix_stride); + map.read(&uni->row_major); + map.read(&uni->atomic_buffer_index); + + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) { + map.read(&uni->sampler[i].index); + map.read(&uni->sampler[i].active); + } + + uint32_t size; + map.read(&size); + + CACHE_DEBUG("read uniform storage size %ld\n", + size * sizeof(union gl_constant_value)); + + uni->storage = + rzalloc_array(mem_ctx, union gl_constant_value, size); + + map.read(uni->storage, size * sizeof(union gl_constant_value)); + + /* driver uniform storage gets generated and propagated later */ + uni->driver_storage = NULL; + uni->num_driver_storage = 0; +} + + +/** + * Deserialize gl_shader_program structure + */ +extern "C" int +_mesa_program_deserialize(struct gl_shader_program *prog, + const GLvoid *blob, size_t size, const char *mesa_sha) +{ + memory_map map; + map.map((const void*)blob, size); + + map.read(&prog->Type); + map.read(&prog->NumShaders); + map.read(&prog->LinkStatus); + map.read(&prog->Version); + map.read(&prog->IsES); + + prog->NumUserUniformStorage = 0; + prog->UniformStorage = NULL; + prog->Label = NULL; + + map.read(&prog->NumUserUniformStorage); + map.read(&prog->UniformLocationBaseScale); + + /* these already allocated by _mesa_init_shader_program */ + _read_hash_table(prog->AttributeBindings, &map); + _read_hash_table(prog->FragDataBindings, &map); + _read_hash_table(prog->FragDataIndexBindings, &map); + + prog->UniformHash = new string_to_uint_map; + _read_hash_table(prog->UniformHash, &map); + + /* just zero for now */ + prog->LinkedTransformFeedback.Outputs = NULL; + prog->LinkedTransformFeedback.Varyings = NULL; + prog->LinkedTransformFeedback.NumVarying = 0; + prog->LinkedTransformFeedback.NumOutputs = 0; + + /* uniform storage */ + prog->UniformStorage = rzalloc_array(prog, struct gl_uniform_storage, + prog->NumUserUniformStorage); + + for (unsigned i = 0; i < prog->NumUserUniformStorage; i++) + _read_uniform_storage(prog, &prog->UniformStorage[i], map, NULL); + + GET_CURRENT_CONTEXT(ctx); + + /** + * prog->Shaders is not strictly required, however we might want to be + * able to recompile and relink these programs? One disadvantage is that + * it makes the binary blobs a lot bigger + */ </blockquote><div> </div><div>As I commented above, I don't see a reason to store the unlinked shaders. However, if we do decide we want to keep that option open, I'd recommend putting something like this near the top of the file: </div><div>const bool STORE_UNLINKED_SHADERS = false; </div><div>along with a comment explaining the pros and cons of storing the unlinked shaders. Then, here in the code, use </div><div>if (STORE_UNLINKED_SHADERS) rather than #if 0 That way, if we decide to enable storage of unlinked shaders, there's no danger that we'll accidentally enable it in one place but not the other. </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +#if 0 + /* Shaders array (unlinked */ + prog->Shaders = (struct gl_shader **) + _mesa_realloc(prog->Shaders, 0, + (prog->NumShaders) * sizeof(struct gl_shader *)); + + for (unsigned i = 0; i < prog->NumShaders; i++) { + + struct gl_shader *sha = map.read_shader(prog, mesa_sha); + + if (sha) { + prog->Shaders[i] = NULL; /* alloc did not initialize */ + _mesa_reference_shader(ctx, &prog->Shaders[i], sha); + CACHE_DEBUG("%s: read unlinked shader, index %d (%p) size %d\n", + __func__, i, sha, shader_size); + } + } +#else + prog->Shaders = NULL; + prog->NumShaders = 0; +#endif + + /* init list, cache can contain only some shader types */ + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) + prog->_LinkedShaders[i] = NULL; + + /* read _LinkedShaders */ + while(map.position() < size) { + unsigned index; + map.read(&index); + + struct gl_shader *sha = map.read_shader(prog, mesa_sha); + + if (!sha) { + CACHE_DEBUG("failed to read shader (index %d)\n", index); + return -1; + } + +#if 0 </blockquote><div> </div><div>This should also use "if (false)" </div><div> </div><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> + { + GET_CURRENT_CONTEXT(ctx); + _mesa_glsl_parse_state *state = + new(sha) _mesa_glsl_parse_state(ctx, sha->Type, sha); + printf("\n"); + _mesa_print_ir(sha->ir, state); + printf("\n"); + } +#endif + + _mesa_reference_shader(ctx, &prog->_LinkedShaders[index], sha); + CACHE_DEBUG("%s: read a linked shader, index %d (%p)\n", + __func__, index, sha); + } + + return 0; +} diff --git a/src/glsl/ir_cache.h b/src/glsl/ir_cache.h new file mode 100644 index 0000000..0146b48 --- /dev/null +++ b/src/glsl/ir_cache.h @@ -0,0 +1,58 @@ +/* -*- c++ -*- */ +/* + * Copyright © 2013 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once +#ifndef IR_CACHE_H +#define IR_CACHE_H + +/* cache specific debug output */ +#ifdef SHADER_CACHE_DEBUG +#define CACHE_DEBUG(fmt, ...) printf(fmt, ## __VA_ARGS__) +#else +#define CACHE_DEBUG(fmt, ...) do {} while (0) +#endif + +/* C API for the cache */ +#ifdef __cplusplus +extern "C" { +#endif + +char *_mesa_shader_serialize(struct gl_shader *shader, + struct _mesa_glsl_parse_state *state, + const char *mesa_sha, size_t *size); + +struct gl_shader *_mesa_shader_deserialize(void *mem_ctx, + void *blob, const char *mesa_sha, size_t size); + +char *_mesa_program_serialize(struct gl_shader_program *prog, + size_t *size, const char *mesa_sha); + +int _mesa_program_deserialize(struct gl_shader_program *prog, + const GLvoid *blob, size_t size, const char *mesa_sha); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* IR_CACHE_H */ -- 1.8.1.4 </blockquote></div> </div></div>