[Mesa-dev] [PATCH 3/7] glsl: Add initial functions to implement an on-disk cache
Carl Worth
cworth at cworth.org
Wed Feb 4 13:52:57 PST 2015
From: Kristian Høgsberg <krh at bitplanet.net>
This code provides for an on-disk cache of objects. Objects are stored
and retrieved (in ~/.cache/mesa) via names that are arbitrary 20-byte
sequences, (intended to be SHA-1 hashes of something identifying for
the content).
The cache is limited to a maximum number of entries (1024 in this
patch), and uses random replacement. These attributes are managed via
an index file that is stored in the cache directory and mmapped. This
file is indexed by the low-order bytes of the cached object's names
and each entry stores the complete name. So a quick comparison of the
index entry verifies whether the cache has an item, or whether an
existing item should be replaced.
Note: Some FIXME comments are still present in this commit. These will
be addressed in subsequent commits, (and before any of this code gets
any active use).
---
src/glsl/Makefile.am | 4 +
src/glsl/Makefile.sources | 3 +
src/glsl/cache.c | 230 ++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 237 insertions(+)
create mode 100644 src/glsl/cache.c
diff --git a/src/glsl/Makefile.am b/src/glsl/Makefile.am
index 01123bc..604af51 100644
--- a/src/glsl/Makefile.am
+++ b/src/glsl/Makefile.am
@@ -137,6 +137,10 @@ libglsl_la_SOURCES = \
$(LIBGLSL_FILES) \
$(NIR_FILES)
+if ENABLE_SHADER_CACHE
+libglsl_la_SOURCES += $(LIBGLSL_SHADER_CACHE_FILES)
+endif
+
glsl_compiler_SOURCES = \
$(top_srcdir)/src/mesa/main/imports.c \
$(top_srcdir)/src/mesa/program/prog_hash_table.c \
diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources
index 8375f6e..c5b742c 100644
--- a/src/glsl/Makefile.sources
+++ b/src/glsl/Makefile.sources
@@ -179,6 +179,9 @@ LIBGLSL_FILES = \
$(GLSL_SRCDIR)/s_expression.cpp \
$(GLSL_SRCDIR)/s_expression.h
+LIBGLSL_SHADER_CACHE_FILES = \
+ $(GLSL_SRCDIR)/cache.c
+
# glsl_compiler
GLSL_COMPILER_CXX_FILES = \
diff --git a/src/glsl/cache.c b/src/glsl/cache.c
new file mode 100644
index 0000000..fd087db
--- /dev/null
+++ b/src/glsl/cache.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <pwd.h>
+
+#include "util/mesa-sha1.h"
+
+#include "cache.h"
+
+#define INDEX_SIZE 1024
+struct program_cache {
+ unsigned char *index;
+ char path[256];
+};
+
+struct program_cache *
+cache_create(void)
+{
+ struct program_cache *cache;
+ char index_file[256], buffer[512];
+ struct stat sb;
+ size_t size;
+ int fd;
+ struct passwd pwd, *result;
+
+ getpwuid_r(getuid(), &pwd, buffer, sizeof buffer, &result);
+ if (result == NULL)
+ return NULL;
+ snprintf(index_file, sizeof index_file,
+ "%s/.cache/mesa/index", pwd.pw_dir);
+
+ fd = open(index_file, O_RDWR | O_CREAT | O_CLOEXEC, 0644);
+ if (fd == -1) {
+ /* FIXME: Check for ENOENT and mkdir on demand */
+ return NULL;
+ }
+
+ if (fstat(fd, &sb) == -1) {
+ close(fd);
+ return NULL;
+ }
+
+ size = INDEX_SIZE * CACHE_KEY_SIZE;
+ if (sb.st_size == 0) {
+ if (ftruncate(fd, size) == -1) {
+ close(fd);
+ return NULL;
+ }
+ fsync(fd);
+ }
+
+ cache = (struct program_cache *) malloc(sizeof *cache);
+ if (cache == NULL) {
+ close(fd);
+ return NULL;
+ }
+
+ snprintf(cache->path, sizeof cache->path,
+ "%s/.cache/mesa", pwd.pw_dir);
+
+ /* FIXME: We map this shared, which is a start, but we need to think about
+ * how to make it multi-process safe. */
+ cache->index = (unsigned char *)
+ mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ close(fd);
+ if (cache->index == MAP_FAILED) {
+ free(cache);
+ return NULL;
+ }
+
+ return cache;
+}
+
+static char *
+get_cache_file(struct program_cache *cache,
+ char *buffer, size_t size, cache_key key)
+{
+ char buf[41];
+
+ snprintf(buffer, size, "%s/%s",
+ cache->path, _mesa_sha1_format(buf, key));
+
+ return buffer;
+}
+
+int
+cache_has(struct program_cache *cache, cache_key key)
+{
+ uint32_t *p = (uint32_t *) key;
+ int i = *p & (INDEX_SIZE - 1);
+ unsigned char *entry;
+
+ /* This function lets us test whether a given item is in the cache or not,
+ * without using syscalls or hitting the disk. It's not race free, but the
+ * races are benign. We could get a positive answer but fail to open in
+ * cache_lookup() because we raced with somebody deleting it. That just
+ * means we have to re-compile eventually. Or we could race with somebody
+ * adding it to the cache, in which case we do an unecessary compile. */
+
+ entry = &cache->index[i * CACHE_KEY_SIZE];
+
+ return memcmp(entry, key, CACHE_KEY_SIZE) == 0;
+}
+
+uint8_t *
+cache_get(struct program_cache *cache, cache_key key, size_t *size)
+{
+ int fd, ret, len;
+ struct stat sb;
+ char filename[256], *data;
+
+ if (size)
+ *size = 0;
+
+ if (!cache_has(cache, key))
+ return NULL;
+
+ get_cache_file(cache, filename, sizeof filename, key);
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ if (fd == -1)
+ return NULL;
+
+ if (fstat(fd, &sb) == -1) {
+ close(fd);
+ return NULL;
+ }
+
+ data = (char *) malloc(sb.st_size);
+ if (data == NULL) {
+ close(fd);
+ return NULL;
+ }
+
+ for (len = 0; len < sb.st_size; len += ret) {
+ ret = read(fd, data + len, sb.st_size - len);
+ if (ret == -1) {
+ free(data);
+ close(fd);
+ return NULL;
+ }
+ }
+
+ close(fd);
+
+ if (size)
+ *size = sb.st_size;
+
+ return (void *) data;
+}
+
+void
+cache_put(struct program_cache *cache, cache_key key, const void *data, size_t size)
+{
+ uint32_t *s = (uint32_t *) key;
+ int i = *s & (INDEX_SIZE - 1);
+ unsigned char *entry;
+ int fd, ret;
+ size_t len;
+ char filename[256];
+ const char *p = (const char *) data;
+
+ /* FIXME: We'll need an fsync here and think about races... maybe even need
+ * an flock to avoid leaking files. Or maybe fsync, then read back and
+ * verify the entry is still ours, delete it if somebody else overwrote
+ * it. */
+
+ entry = &cache->index[i * CACHE_KEY_SIZE];
+ get_cache_file(cache, filename, sizeof filename, entry);
+ unlink(filename);
+ memcpy(entry, key, CACHE_KEY_SIZE);
+
+ if (data == NULL)
+ return;
+
+ /* FIXME: We should write the file to a name like <sha1>-foo, close it and
+ * then rename(2) it to <sha1> to make sure some other mesa process doesn't
+ * open it and gets a partial result. Racing with another mesa writing the
+ * same file is ok, since they'll both write the same contents, and whoever
+ * finishes first will move the complete file in place. */
+
+ get_cache_file(cache, filename, sizeof filename, key);
+ fd = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT, 0644);
+ if (fd == -1)
+ return;
+
+ for (len = 0; len < size; len += ret) {
+ ret = write(fd, p + len, size - len);
+ if (ret == -1) {
+ unlink(filename);
+ break;
+ }
+ }
+
+ close(fd);
+}
+
+void
+cache_mark(struct program_cache *cache, cache_key key)
+{
+ return cache_put(cache, key, NULL, 0);
+}
--
2.1.4
More information about the mesa-dev
mailing list