[Mesa-dev] [v2 5/6] glsl: functions to serialize gl_shader and gl_shader_program
Paul Berry
stereotype441 at gmail.com
Tue Nov 5 14:02:18 PST 2013
On 1 November 2013 02:16, Tapani Pälli <tapani.palli at intel.com> wrote:
> These utility functions can be used to (de)serialize gl_shader and
> gl_shader_program structures. This makes it possible to implement a
> shader compiler cache for individual shaders and functionality required
> by OES_get_program_binary extension.
>
> Signed-off-by: Tapani Pälli <tapani.palli at intel.com>
> ---
> src/glsl/Makefile.sources | 1 +
> src/glsl/ir_cache.cpp | 373
> ++++++++++++++++++++++++++++++++++++++++++++++
> src/glsl/ir_cache.h | 58 +++++++
> 3 files changed, 432 insertions(+)
> create mode 100644 src/glsl/ir_cache.cpp
> create mode 100644 src/glsl/ir_cache.h
>
> diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources
> index 81d5753..99b3c1a 100644
> --- a/src/glsl/Makefile.sources
> +++ b/src/glsl/Makefile.sources
> @@ -30,6 +30,7 @@ LIBGLSL_FILES = \
> $(GLSL_SRCDIR)/hir_field_selection.cpp \
> $(GLSL_SRCDIR)/ir_basic_block.cpp \
> $(GLSL_SRCDIR)/ir_builder.cpp \
> + $(GLSL_SRCDIR)/ir_cache.cpp \
> $(GLSL_SRCDIR)/ir_cache_serializer.cpp \
> $(GLSL_SRCDIR)/ir_cache_deserializer.cpp \
> $(GLSL_SRCDIR)/ir_clone.cpp \
> diff --git a/src/glsl/ir_cache.cpp b/src/glsl/ir_cache.cpp
> new file mode 100644
> index 0000000..24e1c77
> --- /dev/null
> +++ b/src/glsl/ir_cache.cpp
> @@ -0,0 +1,373 @@
> +/* -*- c++ -*- */
> +/*
> + * Copyright © 2013 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"),
> + * to deal in the Software without restriction, including without
> limitation
> + * the rights to use, copy, modify, merge, publish, distribute,
> sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the
> next
> + * paragraph) shall be included in all copies or substantial portions of
> the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
> SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
> OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> + * DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include "main/shaderobj.h"
> +#include "main/uniforms.h"
> +#include "main/macros.h"
> +
> +#include "ir_cache_serializer.h"
> +#include "ir_cache_deserializer.h"
> +
> +/**
> + * Serialize gl_shader structure
> + */
> +extern "C" char *
> +_mesa_shader_serialize(struct gl_shader *shader,
> + struct _mesa_glsl_parse_state *state,
> + const char *mesa_sha, size_t *size)
> +{
> + ir_serializer s;
> + return s.serialize(shader, state, mesa_sha, size);
> +}
> +
> +
> +static void
> +calc_item(const void *key, void *data, void *closure)
>
How about "increment_count" for the name of this function?
> +{
> + unsigned *sz = (unsigned *) closure;
> + *sz = *sz + 1;
> +}
> +
> +
> +static unsigned
> +_hash_table_size(struct string_to_uint_map *map)
>
This function is global, so its name can't start with an underscore--such
names are reserved for library functions. Same goes for many of the other
functions in this file.
> +{
> + unsigned size = 0;
> + map->iterate(calc_item, &size);
> + return size;
> +}
> +
> +
> +static void
> +serialize_item(const void *key, void *data, void *closure)
> +{
> + memory_writer *blob = (memory_writer *) closure;
> + uint32_t value = ((intptr_t)data);
> +
> + blob->write_string((char *)key);
> + blob->write_uint32(&value);
> +}
> +
> +
> +static void
> +_serialize_hash_table(struct string_to_uint_map *map, memory_writer *blob)
> +{
> + uint32_t size = _hash_table_size(map);
> + blob->write_uint32(&size);
> + map->iterate(serialize_item, blob);
>
Rather than take two passes over the hash table (one to compute its size
and one to output its contents), why not do the trick that we do with
overwrite() in ir_cache_serializer.cpp? (In other words, reserve space for
the size of the hashtable in bytes, then serialize the hashtable, then
overwrite the reserved space with the actual size).
> +}
> +
> +
> +static void
> +_serialize_uniform_storage(gl_uniform_storage *uni, memory_writer &blob)
> +{
> + blob.write_string(uni->name);
> +
> + save_glsl_type(blob, uni->type);
> +
> + uint8_t initialized = uni->initialized;
> + uint8_t row_major = uni->row_major;
> +
> + blob.write_uint32(&uni->array_elements);
> + blob.write_uint8(&initialized);
> + blob.write_int32(&uni->block_index);
> + blob.write_int32(&uni->offset);
> + blob.write_int32(&uni->matrix_stride);
> + blob.write_uint8(&row_major);
> + blob.write_int32(&uni->atomic_buffer_index);
> +
> + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
> + uint8_t active = uni->sampler[i].active;
> + blob.write_uint8(&uni->sampler[i].index);
> + blob.write_uint8(&active);
> + }
> +
> + const unsigned elements = MAX2(1, uni->array_elements);
> + const unsigned data_components = elements * uni->type->components();
> + uint32_t size = elements * MAX2(1, data_components);
> +
> + CACHE_DEBUG("%s: size %ld\n", __func__,
> + size * sizeof(union gl_constant_value));
> +
> + blob.write_uint32(&size);
> + blob.write(uni->storage, size * sizeof(union gl_constant_value));
> +}
> +
> +
> +/**
> + * Serialize gl_shader_program structure
> + */
> +extern "C" char *
> +_mesa_program_serialize(struct gl_shader_program *prog, size_t *size,
> + const char *mesa_sha)
> +{
> + memory_writer blob;
> +
> + blob.write_uint32(&prog->Type);
> + blob.write_uint32(&prog->NumShaders);
> + blob.write_uint8(&prog->LinkStatus);
> + blob.write_uint32(&prog->Version);
> + blob.write_uint8(&prog->IsES);
> + blob.write_uint32(&prog->NumUserUniformStorage);
> + blob.write_uint32(&prog->UniformLocationBaseScale);
> +
> + /* hash tables */
> + _serialize_hash_table(prog->AttributeBindings, &blob);
> + _serialize_hash_table(prog->FragDataBindings, &blob);
> + _serialize_hash_table(prog->FragDataIndexBindings, &blob);
> + _serialize_hash_table(prog->UniformHash, &blob);
> +
> + /* uniform storage */
> + if (prog->UniformStorage) {
> + for (unsigned i = 0; i < prog->NumUserUniformStorage; ++i)
> + _serialize_uniform_storage(&prog->UniformStorage[i], blob);
> + }
> +
> + /* Shaders IR, to be decided if we want these to be available */
>
Has there been previous discussion on the mesa-dev list about whether or
not we want these to be available? If so, please include a link to
previous discussion and a brief summary. if not, what does the decision
hinge on? At first blush it seems to me that there's no reason to
serialize the IR of the individual shaders, since ARB_get_program_binary
operates on full programs rather than individual shaders.
Also, typically we prefer to disable code using "if (false)" rather than
ifdefs, because that ensures that the disabled code still compiles.
(Otherwise, it's much more likely to bit rot as the rest of the code base
evolves).
> +#if 0
> + for (unsigned i = 0; i < prog->NumShaders; i++) {
> + size_t sha_size = 0;
> + char *data = _mesa_shader_serialize(prog->Shaders[i],
> + NULL, mesa_sha, &sha_size);
> +
> + if (data) {
> + blob.write(data, sha_size);
> + free(data);
> + }
> + }
> +#endif
> +
> + /* _LinkedShaders IR */
> + for (uint32_t i = 0; i < MESA_SHADER_TYPES; i++) {
> + size_t sha_size = 0;
> +
> + if (!prog->_LinkedShaders[i])
> + continue;
> +
> + char *data = _mesa_shader_serialize(prog->_LinkedShaders[i],
> + NULL, mesa_sha, &sha_size);
> +
> + if (!data) {
> + CACHE_DEBUG("error serializing data for index %d\n", i);
> + return NULL;
> + }
> +
> + /* index in _LinkedShaders list + shader blob */
> + if (data) {
> + blob.write_uint32(&i);
> + blob.write(data, sha_size);
> + free(data);
> + }
> + }
>
Are we going to attempt to store the back-end compiled shaders? It seems
like we'd have to do that in order to get a lot of the benefit of
ARB_get_program_binary.
> +
> + *size = blob.position();
> + return blob.release_memory(size);
> +}
> +
> +
> +/**
> + * Deserialize gl_shader structure
> + */
> +extern "C" struct gl_shader *
> +_mesa_shader_deserialize(void *mem_ctx, void *blob,
> + const char *mesa_sha, size_t size)
> +{
> + int error = 0;
> + ir_deserializer s;
> + memory_map map;
> +
> + map.map(blob, size);
> +
> + return s.deserialize(mem_ctx, map, size,
> + NULL,
> + mesa_sha,
> + &error);
> +}
> +
> +
> +static void
> +_read_hash_table(struct string_to_uint_map *hash, memory_map *map)
> +{
> + unsigned size;
> + map->read(&size);
> +
> + for (unsigned i = 0; i < size; i++) {
> + unsigned value;
> +
> + char *key = map->read_string();
> + map->read(&value);
> +
> + hash->put(value-1, key);
> + }
> +}
> +
> +
> +static void
> +_read_uniform_storage(void *mem_ctx, gl_uniform_storage *uni,
> + memory_map &map, struct _mesa_glsl_parse_state *state)
> +{
> + ir_deserializer s;
> +
> + char *name = map.read_string();
> + uni->name = strdup(name);
> +
> + uni->type = s.read_glsl_type(map, NULL);
> +
> + map.read(&uni->array_elements);
> + map.read(&uni->initialized);
> + map.read(&uni->block_index);
> + map.read(&uni->offset);
> + map.read(&uni->matrix_stride);
> + map.read(&uni->row_major);
> + map.read(&uni->atomic_buffer_index);
> +
> + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
> + map.read(&uni->sampler[i].index);
> + map.read(&uni->sampler[i].active);
> + }
> +
> + uint32_t size;
> + map.read(&size);
> +
> + CACHE_DEBUG("read uniform storage size %ld\n",
> + size * sizeof(union gl_constant_value));
> +
> + uni->storage =
> + rzalloc_array(mem_ctx, union gl_constant_value, size);
> +
> + map.read(uni->storage, size * sizeof(union gl_constant_value));
> +
> + /* driver uniform storage gets generated and propagated later */
> + uni->driver_storage = NULL;
> + uni->num_driver_storage = 0;
> +}
> +
> +
> +/**
> + * Deserialize gl_shader_program structure
> + */
> +extern "C" int
> +_mesa_program_deserialize(struct gl_shader_program *prog,
> + const GLvoid *blob, size_t size, const char *mesa_sha)
> +{
> + memory_map map;
> + map.map((const void*)blob, size);
> +
> + map.read(&prog->Type);
> + map.read(&prog->NumShaders);
> + map.read(&prog->LinkStatus);
> + map.read(&prog->Version);
> + map.read(&prog->IsES);
> +
> + prog->NumUserUniformStorage = 0;
> + prog->UniformStorage = NULL;
> + prog->Label = NULL;
> +
> + map.read(&prog->NumUserUniformStorage);
> + map.read(&prog->UniformLocationBaseScale);
> +
> + /* these already allocated by _mesa_init_shader_program */
> + _read_hash_table(prog->AttributeBindings, &map);
> + _read_hash_table(prog->FragDataBindings, &map);
> + _read_hash_table(prog->FragDataIndexBindings, &map);
> +
> + prog->UniformHash = new string_to_uint_map;
> + _read_hash_table(prog->UniformHash, &map);
> +
> + /* just zero for now */
> + prog->LinkedTransformFeedback.Outputs = NULL;
> + prog->LinkedTransformFeedback.Varyings = NULL;
> + prog->LinkedTransformFeedback.NumVarying = 0;
> + prog->LinkedTransformFeedback.NumOutputs = 0;
> +
> + /* uniform storage */
> + prog->UniformStorage = rzalloc_array(prog, struct gl_uniform_storage,
> + prog->NumUserUniformStorage);
> +
> + for (unsigned i = 0; i < prog->NumUserUniformStorage; i++)
> + _read_uniform_storage(prog, &prog->UniformStorage[i], map, NULL);
> +
> + GET_CURRENT_CONTEXT(ctx);
> +
> + /**
> + * prog->Shaders is not strictly required, however we might want to be
> + * able to recompile and relink these programs? One disadvantage is
> that
> + * it makes the binary blobs a lot bigger
> + */
>
As I commented above, I don't see a reason to store the unlinked shaders.
However, if we do decide we want to keep that option open, I'd recommend
putting something like this near the top of the file:
const bool STORE_UNLINKED_SHADERS = false;
along with a comment explaining the pros and cons of storing the unlinked
shaders. Then, here in the code, use
if (STORE_UNLINKED_SHADERS)
rather than
#if 0
That way, if we decide to enable storage of unlinked shaders, there's no
danger that we'll accidentally enable it in one place but not the other.
> +#if 0
> + /* Shaders array (unlinked */
> + prog->Shaders = (struct gl_shader **)
> + _mesa_realloc(prog->Shaders, 0,
> + (prog->NumShaders) * sizeof(struct gl_shader *));
> +
> + for (unsigned i = 0; i < prog->NumShaders; i++) {
> +
> + struct gl_shader *sha = map.read_shader(prog, mesa_sha);
> +
> + if (sha) {
> + prog->Shaders[i] = NULL; /* alloc did not initialize */
> + _mesa_reference_shader(ctx, &prog->Shaders[i], sha);
> + CACHE_DEBUG("%s: read unlinked shader, index %d (%p) size %d\n",
> + __func__, i, sha, shader_size);
> + }
> + }
> +#else
> + prog->Shaders = NULL;
> + prog->NumShaders = 0;
> +#endif
> +
> + /* init list, cache can contain only some shader types */
> + for (unsigned i = 0; i < MESA_SHADER_TYPES; i++)
> + prog->_LinkedShaders[i] = NULL;
> +
> + /* read _LinkedShaders */
> + while(map.position() < size) {
> + unsigned index;
> + map.read(&index);
> +
> + struct gl_shader *sha = map.read_shader(prog, mesa_sha);
> +
> + if (!sha) {
> + CACHE_DEBUG("failed to read shader (index %d)\n", index);
> + return -1;
> + }
> +
> +#if 0
>
This should also use "if (false)"
> + {
> + GET_CURRENT_CONTEXT(ctx);
> + _mesa_glsl_parse_state *state =
> + new(sha) _mesa_glsl_parse_state(ctx, sha->Type, sha);
> + printf("\n");
> + _mesa_print_ir(sha->ir, state);
> + printf("\n");
> + }
> +#endif
> +
> + _mesa_reference_shader(ctx, &prog->_LinkedShaders[index], sha);
> + CACHE_DEBUG("%s: read a linked shader, index %d (%p)\n",
> + __func__, index, sha);
> + }
> +
> + return 0;
> +}
> diff --git a/src/glsl/ir_cache.h b/src/glsl/ir_cache.h
> new file mode 100644
> index 0000000..0146b48
> --- /dev/null
> +++ b/src/glsl/ir_cache.h
> @@ -0,0 +1,58 @@
> +/* -*- c++ -*- */
> +/*
> + * Copyright © 2013 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the
> "Software"),
> + * to deal in the Software without restriction, including without
> limitation
> + * the rights to use, copy, modify, merge, publish, distribute,
> sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the
> next
> + * paragraph) shall be included in all copies or substantial portions of
> the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
> SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
> OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> + * DEALINGS IN THE SOFTWARE.
> + */
> +
> +#pragma once
> +#ifndef IR_CACHE_H
> +#define IR_CACHE_H
> +
> +/* cache specific debug output */
> +#ifdef SHADER_CACHE_DEBUG
> +#define CACHE_DEBUG(fmt, ...) printf(fmt, ## __VA_ARGS__)
> +#else
> +#define CACHE_DEBUG(fmt, ...) do {} while (0)
> +#endif
> +
> +/* C API for the cache */
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +char *_mesa_shader_serialize(struct gl_shader *shader,
> + struct _mesa_glsl_parse_state *state,
> + const char *mesa_sha, size_t *size);
> +
> +struct gl_shader *_mesa_shader_deserialize(void *mem_ctx,
> + void *blob, const char *mesa_sha, size_t size);
> +
> +char *_mesa_program_serialize(struct gl_shader_program *prog,
> + size_t *size, const char *mesa_sha);
> +
> +int _mesa_program_deserialize(struct gl_shader_program *prog,
> + const GLvoid *blob, size_t size, const char *mesa_sha);
> +
> +#ifdef __cplusplus
> +} /* extern "C" */
> +#endif
> +
> +#endif /* IR_CACHE_H */
> --
> 1.8.1.4
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.freedesktop.org/archives/mesa-dev/attachments/20131105/7a9a2357/attachment-0001.html>
More information about the mesa-dev
mailing list