[Mesa-dev] [PATCH V3 1/5] util/disk_cache: rename mesa cache dir and introduce cache versioning

Dieter Nützel Dieter at nuetzel-hh.de
Fri Aug 25 02:56:15 UTC 2017


For the series:

Tested-by: Dieter Nützel <Dieter at nuetzel-hh.de>

on RX580 with Nine

Dieter

Am 24.08.2017 03:11, schrieb Timothy Arceri:
> Steam is already analysing cache items, unfortunatly we did not
> introduce a versioning mechanism for identifying structural changes
> to cache entries earlier so the only way to do so is to rename the
> cache directory.
> 
> Since we are renaming it we take the opportunity to give the directory
> a more meaningful name.
> 
> Adding a version field to the header of cache entries will help us to
> avoid having to rename the directory in future. Please note this is
> versioning for the internal structure of the entries as defined in
> disk_cache.{c,h} as opposed to the structure of the data provided to
> the disk cache by the GLSL compiler and the various driver backends.
> 
> V3: fix silly bug where cache->driver_keys_blob was incremented 
> directly
> ---
>  src/compiler/glsl/tests/cache_test.c |  6 +++--
>  src/util/disk_cache.c                | 47 
> +++++++++++++++++++++++++++---------
>  src/util/disk_cache.h                |  2 ++
>  3 files changed, 41 insertions(+), 14 deletions(-)
> 
> diff --git a/src/compiler/glsl/tests/cache_test.c
> b/src/compiler/glsl/tests/cache_test.c
> index af1b66fb3d..3796ce6170 100644
> --- a/src/compiler/glsl/tests/cache_test.c
> +++ b/src/compiler/glsl/tests/cache_test.c
> @@ -178,38 +178,40 @@ test_disk_cache_create(void)
>     /* Test with XDG_CACHE_HOME set */
>     setenv("XDG_CACHE_HOME", CACHE_TEST_TMP "/xdg-cache-home", 1);
>     cache = disk_cache_create("test", "make_check", 0);
>     expect_null(cache, "disk_cache_create with XDG_CACHE_HOME set with"
>                 "a non-existing parent directory");
> 
>     mkdir(CACHE_TEST_TMP, 0755);
>     cache = disk_cache_create("test", "make_check", 0);
>     expect_non_null(cache, "disk_cache_create with XDG_CACHE_HOME 
> set");
> 
> -   check_directories_created(CACHE_TEST_TMP "/xdg-cache-home/mesa");
> +   check_directories_created(CACHE_TEST_TMP "/xdg-cache-home/"
> +                             CACHE_DIR_NAME);
> 
>     disk_cache_destroy(cache);
> 
>     /* Test with MESA_GLSL_CACHE_DIR set */
>     err = rmrf_local(CACHE_TEST_TMP);
>     expect_equal(err, 0, "Removing " CACHE_TEST_TMP);
> 
>     setenv("MESA_GLSL_CACHE_DIR", CACHE_TEST_TMP 
> "/mesa-glsl-cache-dir", 1);
>     cache = disk_cache_create("test", "make_check", 0);
>     expect_null(cache, "disk_cache_create with MESA_GLSL_CACHE_DIR set 
> with"
>                 "a non-existing parent directory");
> 
>     mkdir(CACHE_TEST_TMP, 0755);
>     cache = disk_cache_create("test", "make_check", 0);
>     expect_non_null(cache, "disk_cache_create with MESA_GLSL_CACHE_DIR 
> set");
> 
> -   check_directories_created(CACHE_TEST_TMP 
> "/mesa-glsl-cache-dir/mesa");
> +   check_directories_created(CACHE_TEST_TMP "/mesa-glsl-cache-dir/"
> +                             CACHE_DIR_NAME);
> 
>     disk_cache_destroy(cache);
>  }
> 
>  static bool
>  does_cache_contain(struct disk_cache *cache, const cache_key key)
>  {
>     void *result;
> 
>     result = disk_cache_get(cache, key, NULL);
> diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
> index b2229874e0..b2747fbce4 100644
> --- a/src/util/disk_cache.c
> +++ b/src/util/disk_cache.c
> @@ -51,20 +51,34 @@
> 
>  /* Number of bits to mask off from a cache key to get an index. */
>  #define CACHE_INDEX_KEY_BITS 16
> 
>  /* Mask for computing an index from a key. */
>  #define CACHE_INDEX_KEY_MASK ((1 << CACHE_INDEX_KEY_BITS) - 1)
> 
>  /* The number of keys that can be stored in the index. */
>  #define CACHE_INDEX_MAX_KEYS (1 << CACHE_INDEX_KEY_BITS)
> 
> +/* The cache version should be bumped whenever a change is made to the
> + * structure of cache entries or the index. This will give any 3rd 
> party
> + * applications reading the cache entries a chance to adjust to the 
> changes.
> + *
> + * - The cache version is checked internally when reading a cache 
> entry. If we
> + *   ever have a mismatch we are in big trouble as this means we had a 
> cache
> + *   collision. In case of such an event please check the skys for 
> giant
> + *   asteroids and that the entire Mesa team hasn't been eaten by 
> wolves.
> + *
> + * - There is no strict requirement that cache versions be backwards
> + *   compatible but effort should be taken to limit disruption where 
> possible.
> + */
> +#define CACHE_VERSION 1
> +
>  struct disk_cache {
>     /* The path to the cache directory. */
>     char *path;
> 
>     /* Thread queue for compressing and writing cache entries to disk 
> */
>     struct util_queue cache_queue;
> 
>     /* Seed for rand, which is used to pick a random directory */
>     uint64_t seed_xorshift128plus[2];
> 
> @@ -153,20 +167,25 @@ concatenate_and_mkdir(void *ctx, const char
> *path, const char *name)
>        return NULL;
> 
>     new_path = ralloc_asprintf(ctx, "%s/%s", path, name);
> 
>     if (mkdir_if_needed(new_path) == 0)
>        return new_path;
>     else
>        return NULL;
>  }
> 
> +#define DRV_KEY_CPY(_dst, _src, _src_size) { \
> +   memcpy(_dst, _src, _src_size);            \
> +   _dst += _src_size;                        \
> +} while (0)
> +
>  struct disk_cache *
>  disk_cache_create(const char *gpu_name, const char *timestamp,
>                    uint64_t driver_flags)
>  {
>     void *local;
>     struct disk_cache *cache = NULL;
>     char *path, *max_size_str;
>     uint64_t max_size;
>     int fd = -1;
>     struct stat sb;
> @@ -181,41 +200,41 @@ disk_cache_create(const char *gpu_name, const
> char *timestamp,
>     if (local == NULL)
>        goto fail;
> 
>     /* At user request, disable shader cache entirely. */
>     if (getenv("MESA_GLSL_CACHE_DISABLE"))
>        goto fail;
> 
>     /* Determine path for cache based on the first defined name as 
> follows:
>      *
>      *   $MESA_GLSL_CACHE_DIR
> -    *   $XDG_CACHE_HOME/mesa
> -    *   <pwd.pw_dir>/.cache/mesa
> +    *   $XDG_CACHE_HOME/mesa_shader_cache
> +    *   <pwd.pw_dir>/.cache/mesa_shader_cache
>      */
>     path = getenv("MESA_GLSL_CACHE_DIR");
>     if (path) {
>        if (mkdir_if_needed(path) == -1)
>           goto fail;
> 
> -      path = concatenate_and_mkdir(local, path, "mesa");
> +      path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
>        if (path == NULL)
>           goto fail;
>     }
> 
>     if (path == NULL) {
>        char *xdg_cache_home = getenv("XDG_CACHE_HOME");
> 
>        if (xdg_cache_home) {
>           if (mkdir_if_needed(xdg_cache_home) == -1)
>              goto fail;
> 
> -         path = concatenate_and_mkdir(local, xdg_cache_home, "mesa");
> +         path = concatenate_and_mkdir(local, xdg_cache_home, 
> CACHE_DIR_NAME);
>           if (path == NULL)
>              goto fail;
>        }
>     }
> 
>     if (path == NULL) {
>        char *buf;
>        size_t buf_size;
>        struct passwd pwd, *result;
> 
> @@ -237,21 +256,21 @@ disk_cache_create(const char *gpu_name, const
> char *timestamp,
>              buf_size *= 2;
>           } else {
>              goto fail;
>           }
>        }
> 
>        path = concatenate_and_mkdir(local, pwd.pw_dir, ".cache");
>        if (path == NULL)
>           goto fail;
> 
> -      path = concatenate_and_mkdir(local, path, "mesa");
> +      path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
>        if (path == NULL)
>           goto fail;
>     }
> 
>     cache = ralloc(NULL, struct disk_cache);
>     if (cache == NULL)
>        goto fail;
> 
>     cache->path = ralloc_strdup(cache, path);
>     if (cache->path == NULL)
> @@ -337,47 +356,51 @@ disk_cache_create(const char *gpu_name, const
> char *timestamp,
>     cache->max_size = max_size;
> 
>     /* A limit of 32 jobs was choosen as observations of Deus Ex 
> start-up times
>      * showed that we reached at most 11 jobs on an Intel i5-6400 
> CPU at 2.70GHz
>      * (a fairly modest desktop CPU). 1 thread was chosen because we 
> don't
>      * really care about getting things to disk quickly just that it's 
> not
>      * blocking other tasks.
>      */
>     util_queue_init(&cache->cache_queue, "disk_cache", 32, 1, 0);
> 
> +   uint8_t cache_version = CACHE_VERSION;
> +   size_t cv_size = sizeof(cache_version);
> +   cache->driver_keys_blob_size = cv_size;
> +
>     /* Create driver id keys */
>     size_t ts_size = strlen(timestamp) + 1;
>     size_t gpu_name_size = strlen(gpu_name) + 1;
> -   cache->driver_keys_blob_size = ts_size;
> +   cache->driver_keys_blob_size += ts_size;
>     cache->driver_keys_blob_size += gpu_name_size;
> 
>     /* We sometimes store entire structs that contains a pointers in 
> the cache,
>      * use pointer size as a key to avoid hard to debug issues.
>      */
>     uint8_t ptr_size = sizeof(void *);
>     size_t ptr_size_size = sizeof(ptr_size);
>     cache->driver_keys_blob_size += ptr_size_size;
> 
>     size_t driver_flags_size = sizeof(driver_flags);
>     cache->driver_keys_blob_size += driver_flags_size;
> 
>     cache->driver_keys_blob =
>        ralloc_size(cache, cache->driver_keys_blob_size);
>     if (!cache->driver_keys_blob)
>        goto fail;
> 
> -   memcpy(cache->driver_keys_blob, timestamp, ts_size);
> -   memcpy(cache->driver_keys_blob + ts_size, gpu_name, gpu_name_size);
> -   memcpy(cache->driver_keys_blob + ts_size + gpu_name_size, 
> &ptr_size,
> -          ptr_size_size);
> -   memcpy(cache->driver_keys_blob + ts_size + gpu_name_size + 
> ptr_size_size,
> -          &driver_flags, driver_flags_size);
> +   uint8_t *drv_key_blob = cache->driver_keys_blob;
> +   DRV_KEY_CPY(drv_key_blob, &cache_version, cv_size)
> +   DRV_KEY_CPY(drv_key_blob, timestamp, ts_size)
> +   DRV_KEY_CPY(drv_key_blob, gpu_name, gpu_name_size)
> +   DRV_KEY_CPY(drv_key_blob, &ptr_size, ptr_size_size)
> +   DRV_KEY_CPY(drv_key_blob, &driver_flags, driver_flags_size)
> 
>     /* Seed our rand function */
>     s_rand_xorshift128plus(cache->seed_xorshift128plus, true);
> 
>     ralloc_free(local);
> 
>     return cache;
> 
>   fail:
>     if (fd != -1)
> diff --git a/src/util/disk_cache.h b/src/util/disk_cache.h
> index 9aade16a9e..3521ef4a6b 100644
> --- a/src/util/disk_cache.h
> +++ b/src/util/disk_cache.h
> @@ -32,20 +32,22 @@
>  #include <stdbool.h>
>  #include <sys/stat.h>
> 
>  #ifdef __cplusplus
>  extern "C" {
>  #endif
> 
>  /* Size of cache keys in bytes. */
>  #define CACHE_KEY_SIZE 20
> 
> +#define CACHE_DIR_NAME "mesa_shader_cache"
> +
>  typedef uint8_t cache_key[CACHE_KEY_SIZE];
> 
>  struct disk_cache;
> 
>  static inline bool
>  disk_cache_get_function_timestamp(void *ptr, uint32_t* timestamp)
>  {
>  #ifdef ENABLE_SHADER_CACHE
>     Dl_info info;
>     struct stat st;


More information about the mesa-dev mailing list