[Mesa-dev] [PATCH 3/7] glsl: Add initial functions to implement an on-disk cache

Kenneth Graunke kenneth at whitecape.org
Wed Feb 4 15:31:26 PST 2015


On Wednesday, February 04, 2015 01:52:57 PM Carl Worth wrote:
> From: Kristian Høgsberg <krh at bitplanet.net>
> 
> This code provides for an on-disk cache of objects. Objects are stored
> and retrieved (in ~/.cache/mesa) via names that are arbitrary 20-byte
> sequences, (intended to be SHA-1 hashes of something identifying for
> the content).
> 
> The cache is limited to a maximum number of entries (1024 in this
> patch), and uses random replacement. These attributes are managed via

Hi Carl,

The cache will need to be much larger than 1024 entries - perhaps by an
order of magnitude.  For example, "Shadowrun Returns" uses 1299 shaders,
"Left 4 Dead 2" uses 1849 shaders, and "Natural Selection 2" uses 2719
shaders.  A single application could overflow the cache :)

For what it's worth, the total number of shaders in my game collection
is around 17,000.

A couple of other comments inline...

> an index file that is stored in the cache directory and mmapped. This
> file is indexed by the low-order bytes of the cached object's names
> and each entry stores the complete name. So a quick comparison of the
> index entry verifies whether the cache has an item, or whether an
> existing item should be replaced.
> 
> Note: Some FIXME comments are still present in this commit. These will
> be addressed in subsequent commits, (and before any of this code gets
> any active use).
> ---
>  src/glsl/Makefile.am      |   4 +
>  src/glsl/Makefile.sources |   3 +
>  src/glsl/cache.c          | 230 ++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 237 insertions(+)
>  create mode 100644 src/glsl/cache.c
> 
> diff --git a/src/glsl/Makefile.am b/src/glsl/Makefile.am
> index 01123bc..604af51 100644
> --- a/src/glsl/Makefile.am
> +++ b/src/glsl/Makefile.am
> @@ -137,6 +137,10 @@ libglsl_la_SOURCES =					\
>  	$(LIBGLSL_FILES)				\
>  	$(NIR_FILES)
>  
> +if ENABLE_SHADER_CACHE
> +libglsl_la_SOURCES += $(LIBGLSL_SHADER_CACHE_FILES)
> +endif
> +
>  glsl_compiler_SOURCES = \
>  	$(top_srcdir)/src/mesa/main/imports.c \
>  	$(top_srcdir)/src/mesa/program/prog_hash_table.c \
> diff --git a/src/glsl/Makefile.sources b/src/glsl/Makefile.sources
> index 8375f6e..c5b742c 100644
> --- a/src/glsl/Makefile.sources
> +++ b/src/glsl/Makefile.sources
> @@ -179,6 +179,9 @@ LIBGLSL_FILES = \
>  	$(GLSL_SRCDIR)/s_expression.cpp \
>  	$(GLSL_SRCDIR)/s_expression.h
>  
> +LIBGLSL_SHADER_CACHE_FILES = \
> +	$(GLSL_SRCDIR)/cache.c
> +
>  # glsl_compiler
>  
>  GLSL_COMPILER_CXX_FILES = \
> diff --git a/src/glsl/cache.c b/src/glsl/cache.c
> new file mode 100644
> index 0000000..fd087db
> --- /dev/null
> +++ b/src/glsl/cache.c
> @@ -0,0 +1,230 @@
> +/*
> + * Copyright © 2014 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + */
> +
> +#include <string.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <pwd.h>
> +
> +#include "util/mesa-sha1.h"
> +
> +#include "cache.h"
> +
> +#define INDEX_SIZE 1024
> +struct program_cache {
> +   unsigned char *index;
> +   char path[256];
> +};
> +
> +struct program_cache *
> +cache_create(void)
> +{
> +   struct program_cache *cache;
> +   char index_file[256], buffer[512];
> +   struct stat sb;
> +   size_t size;
> +   int fd;
> +   struct passwd pwd, *result;
> +
> +   getpwuid_r(getuid(), &pwd, buffer, sizeof buffer, &result);
> +   if (result == NULL)
> +      return NULL;
> +   snprintf(index_file, sizeof index_file,
> +            "%s/.cache/mesa/index", pwd.pw_dir);

We should respect $XDG_CACHE_HOME from the XDG base directory spec:
http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html

Also, the snprintf makes me a bit nervous - we don't know how long the
user's home directory path will be, much less $XDG_CACHE_HOME.  snprintf
saves us from overflowing the buffer, but we may try to open/create an
unexpected file.  Perhaps we should malloc it?

> +
> +   fd = open(index_file, O_RDWR | O_CREAT | O_CLOEXEC, 0644);
> +   if (fd == -1) {
> +      /* FIXME: Check for ENOENT and mkdir on demand */
> +      return NULL;
> +   }
> +
> +   if (fstat(fd, &sb) == -1) {
> +      close(fd);
> +      return NULL;
> +   }
> +
> +   size = INDEX_SIZE * CACHE_KEY_SIZE;
> +   if (sb.st_size == 0) {
> +      if (ftruncate(fd, size) == -1) {
> +         close(fd);
> +         return NULL;
> +      }
> +      fsync(fd);
> +   }
> +
> +   cache = (struct program_cache *) malloc(sizeof *cache);
> +   if (cache == NULL) {
> +      close(fd);
> +      return NULL;
> +   }
> +
> +   snprintf(cache->path, sizeof cache->path,
> +            "%s/.cache/mesa", pwd.pw_dir);
> +
> +   /* FIXME: We map this shared, which is a start, but we need to think about
> +    * how to make it multi-process safe. */
> +   cache->index = (unsigned char *)
> +      mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
> +   close(fd);
> +   if (cache->index == MAP_FAILED) {
> +      free(cache);
> +      return NULL;
> +   }
> +
> +   return cache;
> +}
> +
> +static char *
> +get_cache_file(struct program_cache *cache,
> +               char *buffer, size_t size, cache_key key)
> +{
> +   char buf[41];
> +
> +   snprintf(buffer, size, "%s/%s",
> +            cache->path, _mesa_sha1_format(buf, key));
> +
> +   return buffer;
> +}
> +
> +int
> +cache_has(struct program_cache *cache, cache_key key)
> +{
> +   uint32_t *p = (uint32_t *) key;
> +   int i = *p & (INDEX_SIZE - 1);
> +   unsigned char *entry;
> +
> +   /* This function lets us test whether a given item is in the cache or not,
> +    * without using syscalls or hitting the disk.  It's not race free, but the
> +    * races are benign.  We could get a positive answer but fail to open in
> +    * cache_lookup() because we raced with somebody deleting it.  That just
> +    * means we have to re-compile eventually.  Or we could race with somebody
> +    * adding it to the cache, in which case we do an unecessary compile. */

Typo: unnecessary compile (missing 'n').  Also, */ goes on its own line.

> +
> +   entry = &cache->index[i * CACHE_KEY_SIZE];
> +
> +   return memcmp(entry, key, CACHE_KEY_SIZE) == 0;
> +}
> +
> +uint8_t *
> +cache_get(struct program_cache *cache, cache_key key, size_t *size)
> +{
> +   int fd, ret, len;
> +   struct stat sb;
> +   char filename[256], *data;
> +
> +   if (size)
> +      *size = 0;
> +
> +   if (!cache_has(cache, key))
> +      return NULL;
> +
> +   get_cache_file(cache, filename, sizeof filename, key);
> +
> +   fd = open(filename, O_RDONLY | O_CLOEXEC);
> +   if (fd == -1)
> +      return NULL;
> +
> +   if (fstat(fd, &sb) == -1) {
> +      close(fd);
> +      return NULL;
> +   }
> +
> +   data = (char *) malloc(sb.st_size);
> +   if (data == NULL) {
> +      close(fd);
> +      return NULL;
> +   }
> +
> +   for (len = 0; len < sb.st_size; len += ret) {
> +      ret = read(fd, data + len, sb.st_size - len);
> +      if (ret == -1) {
> +         free(data);
> +         close(fd);
> +         return NULL;
> +      }
> +   }
> +
> +   close(fd);
> +
> +   if (size)
> +      *size = sb.st_size;
> +
> +   return (void *) data;
> +}
> +
> +void
> +cache_put(struct program_cache *cache, cache_key key, const void *data, size_t size)
> +{
> +   uint32_t *s = (uint32_t *) key;
> +   int i = *s & (INDEX_SIZE - 1);
> +   unsigned char *entry;
> +   int fd, ret;
> +   size_t len;
> +   char filename[256];
> +   const char *p = (const char *) data;
> +
> +   /* FIXME: We'll need an fsync here and think about races... maybe even need
> +    * an flock to avoid leaking files.  Or maybe fsync, then read back and
> +    * verify the entry is still ours, delete it if somebody else overwrote
> +    * it. */
> +
> +   entry = &cache->index[i * CACHE_KEY_SIZE];
> +   get_cache_file(cache, filename, sizeof filename, entry);
> +   unlink(filename);
> +   memcpy(entry, key, CACHE_KEY_SIZE);
> +
> +   if (data == NULL)
> +      return;
> +
> +   /* FIXME: We should write the file to a name like <sha1>-foo, close it and
> +    * then rename(2) it to <sha1> to make sure some other mesa process doesn't
> +    * open it and gets a partial result.  Racing with another mesa writing the
> +    * same file is ok, since they'll both write the same contents, and whoever
> +    * finishes first will move the complete file in place. */
> +
> +   get_cache_file(cache, filename, sizeof filename, key);
> +   fd = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT, 0644);
> +   if (fd == -1)
> +      return;
> +
> +   for (len = 0; len < size; len += ret) {
> +      ret = write(fd, p + len, size - len);
> +      if (ret == -1) {
> +         unlink(filename);
> +         break;
> +      }
> +   }
> +
> +   close(fd);
> +}
> +
> +void
> +cache_mark(struct program_cache *cache, cache_key key)
> +{
> +   return cache_put(cache, key, NULL, 0);
> +}
> 
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: This is a digitally signed message part.
URL: <http://lists.freedesktop.org/archives/mesa-dev/attachments/20150204/a4edb383/attachment.sig>


More information about the mesa-dev mailing list