[Mesa-dev] [wip 8/9] glsl: functions to serialize gl_shader and gl_shader_program
Tapani Pälli
tapani.palli at intel.com
Thu Jan 2 03:58:20 PST 2014
These utility functions can be used to (de)serialize gl_shader and
gl_shader_program structures. This makes it possible to implement a
shader compiler cache for individual shaders and functionality required
by OES_get_program_binary extension.
Signed-off-by: Tapani Pälli <tapani.palli at intel.com>
---
src/glsl/Makefile.sources | 1 +
src/glsl/shader_cache.cpp | 489 ++++++++++++++++++++++++++++++++++++++++++++++
src/glsl/shader_cache.h | 58 ++++++
3 files changed, 548 insertions(+)
create mode 100644 src/glsl/shader_cache.cpp
create mode 100644 src/glsl/shader_cache.h
diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources
index fd4c15e..188deac 100644
--- a/src/glsl/Makefile.sources
+++ b/src/glsl/Makefile.sources
@@ -102,6 +102,7 @@ LIBGLSL_FILES = \
$(GLSL_SRCDIR)/opt_swizzle_swizzle.cpp \
$(GLSL_SRCDIR)/opt_tree_grafting.cpp \
$(GLSL_SRCDIR)/s_expression.cpp \
+ $(GLSL_SRCDIR)/shader_cache.cpp \
$(GLSL_SRCDIR)/strtod.c
# glsl_compiler
diff --git a/src/glsl/shader_cache.cpp b/src/glsl/shader_cache.cpp
new file mode 100644
index 0000000..4b5de45
--- /dev/null
+++ b/src/glsl/shader_cache.cpp
@@ -0,0 +1,489 @@
+/* -*- c++ -*- */
+/*
+ * Copyright © 2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "main/shaderobj.h"
+#include "main/uniforms.h"
+#include "main/macros.h"
+#include "program/hash_table.h"
+#include "ir_serialize.h"
+#include "ir_deserializer.h"
+#include "shader_cache_magic.h"
+
+/**
+ * It is currently unknown if we need these to be available.
+ * There can be calls in mesa/main (like glGetAttachedShaders) that use
+ * the Shaders list.
+ */
+const bool STORE_UNLINKED_SHADERS = false;
+
+
+static void
+write_header(gl_shader *shader, memory_writer &blob)
+{
+ GET_CURRENT_CONTEXT(ctx);
+
+ blob.write_string(mesa_get_shader_cache_magic());
+ blob.write_string((const char *)ctx->Driver.GetString(ctx, GL_VENDOR));
+ blob.write_string((const char *)ctx->Driver.GetString(ctx, GL_RENDERER));
+ blob.write_uint32_t(shader->Version);
+ blob.write_uint32_t(shader->Type);
+ blob.write_uint8_t(shader->IsES);
+
+ /* post-link data */
+ blob.write_uint32_t(shader->num_samplers);
+ blob.write_uint32_t(shader->active_samplers);
+ blob.write_uint32_t(shader->shadow_samplers);
+ blob.write_uint32_t(shader->num_uniform_components);
+ blob.write_uint32_t(shader->num_combined_uniform_components);
+
+ uint8_t uses_builtins = shader->uses_builtin_functions;
+ blob.write_uint8_t(uses_builtins);
+
+ blob.write(&shader->Geom, sizeof(shader->Geom));
+
+ for (unsigned i = 0; i < MAX_SAMPLERS; i++)
+ blob.write_uint8_t(shader->SamplerUnits[i]);
+
+ for (unsigned i = 0; i < MAX_SAMPLERS; i++) {
+ int32_t target = shader->SamplerTargets[i];
+ blob.write_int32_t(target);
+ }
+}
+
+
+/**
+ * Serializes gl_shader structure, writes shader header
+ * information and exec_list of instructions
+ */
+extern "C" char *
+mesa_shader_serialize(struct gl_shader *shader, size_t *size)
+{
+ *size = 0;
+
+ memory_writer blob;
+
+ int32_t start_pos = blob.position();
+ uint32_t shader_data_len = 0;
+ uint32_t shader_type = shader->Type;
+
+ blob.write_uint32_t(shader_data_len);
+ blob.write_uint32_t(shader_type);
+
+ write_header(shader, blob);
+
+ /* dump all shader instructions */
+ serialize_list(*shader->ir, blob);
+
+ shader_data_len = blob.position() -
+ start_pos - sizeof(shader_data_len);
+ blob.overwrite(&shader_data_len, sizeof(shader_data_len), start_pos);
+
+ return blob.release_memory(size);
+}
+
+
+/**
+ * helper structure for hash serialization, hash size is
+ * counted to item_count during serialization
+ */
+struct hash_serialize_data
+{
+ void *writer;
+ uint32_t item_count;
+};
+
+
+static void
+serialize_hash(const void *key, void *data, void *closure)
+{
+ hash_serialize_data *s_data = (hash_serialize_data *) closure;
+ memory_writer *blob = (memory_writer *) s_data->writer;
+
+ uint32_t value = ((intptr_t)data);
+
+ blob->write_string((char *)key);
+ blob->write_uint32_t(value);
+
+ s_data->item_count++;
+}
+
+
+static void
+serialize_hash_table(struct string_to_uint_map *map, memory_writer *blob)
+{
+ struct hash_serialize_data data = {
+ .writer = blob,
+ .item_count = 0
+ };
+
+ int32_t pos = blob->position();
+ blob->write_uint32_t(data.item_count);
+
+ map->iterate(serialize_hash, &data);
+
+ blob->overwrite(&data.item_count, sizeof(data.item_count), pos);
+}
+
+
+static void
+serialize_uniform_storage(gl_uniform_storage *uni, memory_writer &blob)
+{
+ blob.write_string(uni->name);
+
+ /* note, type is not serialized, it is resolved during parsing */
+
+ uint8_t initialized = uni->initialized;
+ uint8_t row_major = uni->row_major;
+
+ blob.write_uint32_t(uni->array_elements);
+ blob.write_uint8_t(initialized);
+ blob.write_int32_t(uni->block_index);
+ blob.write_int32_t(uni->offset);
+ blob.write_int32_t(uni->matrix_stride);
+ blob.write_uint8_t(row_major);
+ blob.write_int32_t(uni->atomic_buffer_index);
+
+ for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
+ uint8_t active = uni->sampler[i].active;
+ blob.write_uint8_t(uni->sampler[i].index);
+ blob.write_uint8_t(active);
+ }
+
+ /* how many elements (1 if not array) * how many components in the type */
+ const unsigned elements = MAX2(1, uni->array_elements);
+ uint32_t size = elements * uni->type->components();
+
+ CACHE_DEBUG("%s: size %ld\n", __func__,
+ size * sizeof(union gl_constant_value));
+
+ blob.write_uint32_t(size);
+ blob.write(uni->storage, size * sizeof(union gl_constant_value));
+}
+
+
+/**
+ * Serialize gl_shader_program structure
+ */
+extern "C" char *
+mesa_program_serialize(struct gl_shader_program *prog, size_t *size,
+ const char *mesa_sha)
+{
+ memory_writer blob;
+
+ blob.write_uint32_t(prog->Type);
+ blob.write_uint32_t(prog->NumShaders);
+ blob.write_uint8_t(prog->LinkStatus);
+ blob.write_uint32_t(prog->Version);
+ blob.write_uint8_t(prog->IsES);
+ blob.write_uint32_t(prog->NumUserUniformStorage);
+ blob.write_uint32_t(prog->UniformLocationBaseScale);
+ blob.write_uint32_t(prog->LastClipDistanceArraySize);
+ blob.write_uint8_t(prog->FragDepthLayout);
+
+ /* hash tables */
+ serialize_hash_table(prog->AttributeBindings, &blob);
+ serialize_hash_table(prog->FragDataBindings, &blob);
+ serialize_hash_table(prog->FragDataIndexBindings, &blob);
+ serialize_hash_table(prog->UniformHash, &blob);
+
+ blob.write(&prog->Geom, sizeof(prog->Geom));
+ blob.write(&prog->Vert, sizeof(prog->Vert));
+
+ /* uniform storage */
+ if (prog->UniformStorage) {
+ for (unsigned i = 0; i < prog->NumUserUniformStorage; ++i)
+ serialize_uniform_storage(&prog->UniformStorage[i], blob);
+ }
+
+ /* Shaders IR, to be decided if we want these to be available */
+#if (STORE_UNLINKED_SHADERS)
+ for (unsigned i = 0; i < prog->NumShaders; i++) {
+ size_t sha_size = 0;
+ char *data = mesa_shader_serialize(prog->Shaders[i],
+ NULL, &sha_size);
+
+ if (data) {
+ blob.write(data, sha_size);
+ free(data);
+ }
+ }
+#endif
+
+ /* _LinkedShaders IR */
+ for (uint32_t i = 0; i < MESA_SHADER_TYPES; i++) {
+ size_t sha_size = 0;
+
+ if (!prog->_LinkedShaders[i])
+ continue;
+
+ char *data = mesa_shader_serialize(prog->_LinkedShaders[i], &sha_size);
+
+ if (!data) {
+ CACHE_DEBUG("error serializing data for index %d\n", i);
+ return NULL;
+ }
+
+ /* index in _LinkedShaders list + shader blob */
+ if (data) {
+ blob.write_uint32_t(i);
+ blob.write(data, sha_size);
+ free(data);
+ }
+ }
+
+ *size = blob.position();
+ return blob.release_memory(size);
+}
+
+
+/**
+ * Deserialize gl_shader structure
+ */
+extern "C" struct gl_shader *
+mesa_shader_deserialize(void *mem_ctx, void *blob, size_t size)
+{
+ int error = 0;
+ ir_deserializer s;
+ memory_map map;
+
+ map.map(blob, size);
+
+ return s.deserialize(mem_ctx, &map, size, &error);
+}
+
+
+static void
+read_hash_table(struct string_to_uint_map *hash, memory_map *map)
+{
+ uint32_t size = map->read_uint32_t();
+
+ for (unsigned i = 0; i < size; i++) {
+
+ char *key = map->read_string();
+ uint32_t value = map->read_uint32_t();
+
+ /* put() adds +1 bias on the value (see hash_table.h), this
+ * is taken care here when reading
+ */
+ hash->put(value-1, key);
+ }
+}
+
+
+static void
+read_uniform_storage(void *mem_ctx, gl_uniform_storage *uni, memory_map &map)
+{
+ ir_deserializer s;
+
+ char *name = map.read_string();
+ uni->name = strdup(name);
+
+ /* resolved later */
+ uni->type = NULL;
+
+ uni->array_elements = map.read_uint32_t();
+ uni->initialized = map.read_uint8_t();
+ uni->block_index = map.read_int32_t();
+ uni->offset = map.read_int32_t();
+ uni->matrix_stride = map.read_int32_t();
+ uni->row_major = map.read_uint8_t();
+ uni->atomic_buffer_index = map.read_int32_t();
+
+ for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
+ uni->sampler[i].index = map.read_uint8_t();
+ uni->sampler[i].active = map.read_uint8_t();
+ }
+
+ uint32_t size = map.read_uint32_t();
+
+ CACHE_DEBUG("read uniform storage size %ld\n",
+ size * sizeof(union gl_constant_value));
+
+ uni->storage =
+ rzalloc_array(mem_ctx, union gl_constant_value, size);
+
+ map.read(uni->storage, size * sizeof(union gl_constant_value));
+
+ /* driver uniform storage gets generated and propagated later */
+ uni->driver_storage = NULL;
+ uni->num_driver_storage = 0;
+}
+
+
+static ir_variable *
+search_var(struct exec_list *list, const char *name)
+{
+ foreach_list_safe(node, list) {
+ ir_variable *var = ((ir_instruction *) node)->as_variable();
+ if (var && strcmp(name, var->name) == 0)
+ return var;
+ }
+ return NULL;
+}
+
+
+/**
+ * Resolve glsl_types for uniform_storage
+ */
+static void
+resolve_uniform_types(struct gl_shader_program *prog,
+ struct gl_shader *sha)
+{
+ CACHE_DEBUG("%s\n", __func__);
+
+ /* for each storage, find corresponding uniform from the shader */
+ for (unsigned i = 0; i < prog->NumUserUniformStorage; i++) {
+ ir_variable *var = search_var(sha->ir, prog->UniformStorage[i].name);
+
+ if (var) {
+ CACHE_DEBUG("found uniform %s (setting type %s)\n",
+ prog->UniformStorage[i].name, var->type->name);
+
+ /* for arrays, uniform storage type contains the element type */
+ if (var->type->is_array())
+ prog->UniformStorage[i].type = var->type->element_type();
+ else
+ prog->UniformStorage[i].type = var->type;
+ }
+ }
+}
+
+/* read and serialize a gl_shader */
+static gl_shader *
+read_shader(void *mem_ctx, memory_map &map)
+{
+ uint32_t shader_size = map.read_uint32_t();
+
+ ir_deserializer s;
+ memory_map wrapper;
+ int error;
+
+ /* read existing memory_map with another memory_map */
+ wrapper.map(map, shader_size);
+ struct gl_shader *sha = s.deserialize(mem_ctx,
+ &wrapper, shader_size, &error);
+
+ return sha;
+}
+
+
+/**
+ * Deserialize gl_shader_program structure
+ */
+extern "C" int
+mesa_program_deserialize(struct gl_shader_program *prog,
+ const GLvoid *blob, size_t size)
+{
+ memory_map map;
+ map.map((const void*)blob, size);
+
+ prog->Type = map.read_uint32_t();
+ prog->NumShaders = map.read_uint32_t();
+ prog->LinkStatus = map.read_uint8_t();
+ prog->Version = map.read_uint32_t();
+ prog->IsES = map.read_uint8_t();
+ prog->NumUserUniformStorage = map.read_uint32_t();
+ prog->UniformLocationBaseScale = map.read_uint32_t();
+ prog->LastClipDistanceArraySize = map.read_uint32_t();
+ prog->FragDepthLayout = (gl_frag_depth_layout) map.read_uint8_t();
+
+ prog->UniformStorage = NULL;
+ prog->Label = NULL;
+
+ /* these already allocated by _mesa_init_shader_program */
+ read_hash_table(prog->AttributeBindings, &map);
+ read_hash_table(prog->FragDataBindings, &map);
+ read_hash_table(prog->FragDataIndexBindings, &map);
+
+ prog->UniformHash = new string_to_uint_map;
+ read_hash_table(prog->UniformHash, &map);
+
+ map.read(&prog->Geom, sizeof(prog->Geom));
+ map.read(&prog->Vert, sizeof(prog->Vert));
+
+ /* just zero for now */
+ prog->LinkedTransformFeedback.Outputs = NULL;
+ prog->LinkedTransformFeedback.Varyings = NULL;
+ prog->LinkedTransformFeedback.NumVarying = 0;
+ prog->LinkedTransformFeedback.NumOutputs = 0;
+
+ /* uniform storage */
+ prog->UniformStorage = rzalloc_array(prog, struct gl_uniform_storage,
+ prog->NumUserUniformStorage);
+
+ for (unsigned i = 0; i < prog->NumUserUniformStorage; i++)
+ read_uniform_storage(prog, &prog->UniformStorage[i], map);
+
+ GET_CURRENT_CONTEXT(ctx);
+
+#if (STORE_UNLINKED_SHADERS)
+ /* Shaders array (unlinked */
+ prog->Shaders = (struct gl_shader **)
+ _mesa_realloc(prog->Shaders, 0,
+ (prog->NumShaders) * sizeof(struct gl_shader *));
+
+ for (unsigned i = 0; i < prog->NumShaders; i++) {
+
+ struct gl_shader *sha = read_shader(prog, map);
+
+ if (sha) {
+ prog->Shaders[i] = NULL; /* alloc did not initialize */
+ _mesa_reference_shader(ctx, &prog->Shaders[i], sha);
+ CACHE_DEBUG("%s: read unlinked shader, index %d (%p) size %d\n",
+ __func__, i, sha, shader_size);
+ }
+ }
+#else
+ prog->Shaders = NULL;
+ prog->NumShaders = 0;
+#endif
+
+ /* init list, cache can contain only some shader types */
+ for (unsigned i = 0; i < MESA_SHADER_TYPES; i++)
+ prog->_LinkedShaders[i] = NULL;
+
+ /* read _LinkedShaders */
+ while(map.position() < size) {
+ uint32_t index = map.read_uint32_t();
+
+ struct gl_shader *sha = read_shader(prog, map);
+
+ /* note, for 'automatic cache' in case of failure we would
+ * need to fallback to compiling/linking the shaders in the
+ * prog->Shaders list
+ */
+ if (!sha) {
+ CACHE_DEBUG("failed to read shader (index %d)\n", index);
+ return -1;
+ }
+
+ resolve_uniform_types(prog, sha);
+
+ _mesa_reference_shader(ctx, &prog->_LinkedShaders[index], sha);
+ CACHE_DEBUG("%s: read a linked shader, index %d (%p)\n",
+ __func__, index, sha);
+ }
+
+ return 0;
+}
diff --git a/src/glsl/shader_cache.h b/src/glsl/shader_cache.h
new file mode 100644
index 0000000..b65660d
--- /dev/null
+++ b/src/glsl/shader_cache.h
@@ -0,0 +1,58 @@
+/* -*- c++ -*- */
+/*
+ * Copyright © 2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+#ifndef SHADER_CACHE_H
+#define SHADER_CACHE_H
+
+/* cache specific debug output */
+#ifdef SHADER_CACHE_DEBUG
+#define CACHE_DEBUG(fmt, ...) printf(fmt, ## __VA_ARGS__)
+#else
+#define CACHE_DEBUG(fmt, ...) do {} while (0)
+#endif
+
+/* C API for the cache */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+char *
+mesa_shader_serialize(struct gl_shader *shader, size_t *size);
+
+struct gl_shader *
+mesa_shader_deserialize(void *mem_ctx, void *blob, size_t size);
+
+char *
+mesa_program_serialize(struct gl_shader_program *prog, size_t *size);
+
+int
+mesa_program_deserialize(struct gl_shader_program *prog,
+ const GLvoid *blob, size_t size);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* SHADER_CACHE_H */
--
1.8.3.1
More information about the mesa-dev
mailing list