[Mesa-dev] [PATCH 1/4] util/disk_cache: rename mesa cache dir and introduce cache versioning
Timothy Arceri
tarceri at itsqueeze.com
Fri Aug 18 11:45:21 UTC 2017
On 18/08/17 21:02, Nicolai Hähnle wrote:
> On 15.08.2017 01:26, Timothy Arceri wrote:
>> Steam is already analysing cache items, unfortunatly we did not
>> introduce a versioning mechanism for identifying structural changes
>> to cache entries earlier so the only way to do so is to rename the
>> cache directory.
>>
>> Since we are renaming it we take the opportunity to give the directory
>> a more meaningful name.
>>
>> Adding a version field to the header of cache entries will help us to
>> avoid having to rename the directory in future. Please note this is
>> versioning for the internal structure of the entries as defined in
>> disk_cache.{c,h} as opposed to the structure of the data provided to
>> the disk cache by the GLSL compiler and the various driver backends.
>
> If the version is intended to reflect the disk cache-internal structure
> of files, shouldn't you check the version when loading a file?
Not really we have been relying on cache collisions being improbable up
to this point. We can check but it doesn't add a great deal of value,
the version is also unlikely to change very often.
>
> Also, what about the index file? Shouldn't it get the version as well?
The index can be rebuilt from the from the cache entries once this
series lands. I don't think we need to bother with it.
>
> Cheers,
> Nicolai
>
>
>> ---
>> src/compiler/glsl/tests/cache_test.c | 6 ++++--
>> src/util/disk_cache.c | 37
>> +++++++++++++++++++++++++-----------
>> 2 files changed, 30 insertions(+), 13 deletions(-)
>>
>> diff --git a/src/compiler/glsl/tests/cache_test.c
>> b/src/compiler/glsl/tests/cache_test.c
>> index af1b66fb3d..aa779e3985 100644
>> --- a/src/compiler/glsl/tests/cache_test.c
>> +++ b/src/compiler/glsl/tests/cache_test.c
>> @@ -178,38 +178,40 @@ test_disk_cache_create(void)
>> /* Test with XDG_CACHE_HOME set */
>> setenv("XDG_CACHE_HOME", CACHE_TEST_TMP "/xdg-cache-home", 1);
>> cache = disk_cache_create("test", "make_check", 0);
>> expect_null(cache, "disk_cache_create with XDG_CACHE_HOME set with"
>> "a non-existing parent directory");
>> mkdir(CACHE_TEST_TMP, 0755);
>> cache = disk_cache_create("test", "make_check", 0);
>> expect_non_null(cache, "disk_cache_create with XDG_CACHE_HOME set");
>> - check_directories_created(CACHE_TEST_TMP "/xdg-cache-home/mesa");
>> + check_directories_created(CACHE_TEST_TMP
>> + "/xdg-cache-home/mesa_shader_cache");
>> disk_cache_destroy(cache);
>> /* Test with MESA_GLSL_CACHE_DIR set */
>> err = rmrf_local(CACHE_TEST_TMP);
>> expect_equal(err, 0, "Removing " CACHE_TEST_TMP);
>> setenv("MESA_GLSL_CACHE_DIR", CACHE_TEST_TMP
>> "/mesa-glsl-cache-dir", 1);
>> cache = disk_cache_create("test", "make_check", 0);
>> expect_null(cache, "disk_cache_create with MESA_GLSL_CACHE_DIR
>> set with"
>> "a non-existing parent directory");
>> mkdir(CACHE_TEST_TMP, 0755);
>> cache = disk_cache_create("test", "make_check", 0);
>> expect_non_null(cache, "disk_cache_create with
>> MESA_GLSL_CACHE_DIR set");
>> - check_directories_created(CACHE_TEST_TMP
>> "/mesa-glsl-cache-dir/mesa");
>> + check_directories_created(CACHE_TEST_TMP
>> + "/mesa-glsl-cache-dir/mesa_shader_cache");
>> disk_cache_destroy(cache);
>> }
>> static bool
>> does_cache_contain(struct disk_cache *cache, const cache_key key)
>> {
>> void *result;
>> result = disk_cache_get(cache, key, NULL);
>> diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
>> index b2229874e0..283856b15c 100644
>> --- a/src/util/disk_cache.c
>> +++ b/src/util/disk_cache.c
>> @@ -51,20 +51,28 @@
>> /* Number of bits to mask off from a cache key to get an index. */
>> #define CACHE_INDEX_KEY_BITS 16
>> /* Mask for computing an index from a key. */
>> #define CACHE_INDEX_KEY_MASK ((1 << CACHE_INDEX_KEY_BITS) - 1)
>> /* The number of keys that can be stored in the index. */
>> #define CACHE_INDEX_MAX_KEYS (1 << CACHE_INDEX_KEY_BITS)
>> +#define CACHE_DIR_NAME "mesa_shader_cache"
>> +
>> +/* The cache version should be bumped whenever a change is made to the
>> + * structure of cache entries or the index. This will give any 3rd party
>> + * applications reading the cache entries a chance to adjust to the
>> changes.
>> + */
>> +#define CACHE_VERSION 1
>> +
>> struct disk_cache {
>> /* The path to the cache directory. */
>> char *path;
>> /* Thread queue for compressing and writing cache entries to disk */
>> struct util_queue cache_queue;
>> /* Seed for rand, which is used to pick a random directory */
>> uint64_t seed_xorshift128plus[2];
>> @@ -181,41 +189,41 @@ disk_cache_create(const char *gpu_name, const
>> char *timestamp,
>> if (local == NULL)
>> goto fail;
>> /* At user request, disable shader cache entirely. */
>> if (getenv("MESA_GLSL_CACHE_DISABLE"))
>> goto fail;
>> /* Determine path for cache based on the first defined name as
>> follows:
>> *
>> * $MESA_GLSL_CACHE_DIR
>> - * $XDG_CACHE_HOME/mesa
>> - * <pwd.pw_dir>/.cache/mesa
>> + * $XDG_CACHE_HOME/mesa_shader_cache
>> + * <pwd.pw_dir>/.cache/mesa_shader_cache
>> */
>> path = getenv("MESA_GLSL_CACHE_DIR");
>> if (path) {
>> if (mkdir_if_needed(path) == -1)
>> goto fail;
>> - path = concatenate_and_mkdir(local, path, "mesa");
>> + path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
>> if (path == NULL)
>> goto fail;
>> }
>> if (path == NULL) {
>> char *xdg_cache_home = getenv("XDG_CACHE_HOME");
>> if (xdg_cache_home) {
>> if (mkdir_if_needed(xdg_cache_home) == -1)
>> goto fail;
>> - path = concatenate_and_mkdir(local, xdg_cache_home, "mesa");
>> + path = concatenate_and_mkdir(local, xdg_cache_home,
>> CACHE_DIR_NAME);
>> if (path == NULL)
>> goto fail;
>> }
>> }
>> if (path == NULL) {
>> char *buf;
>> size_t buf_size;
>> struct passwd pwd, *result;
>> @@ -237,21 +245,21 @@ disk_cache_create(const char *gpu_name, const
>> char *timestamp,
>> buf_size *= 2;
>> } else {
>> goto fail;
>> }
>> }
>> path = concatenate_and_mkdir(local, pwd.pw_dir, ".cache");
>> if (path == NULL)
>> goto fail;
>> - path = concatenate_and_mkdir(local, path, "mesa");
>> + path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
>> if (path == NULL)
>> goto fail;
>> }
>> cache = ralloc(NULL, struct disk_cache);
>> if (cache == NULL)
>> goto fail;
>> cache->path = ralloc_strdup(cache, path);
>> if (cache->path == NULL)
>> @@ -337,46 +345,53 @@ disk_cache_create(const char *gpu_name, const
>> char *timestamp,
>> cache->max_size = max_size;
>> /* A limit of 32 jobs was choosen as observations of Deus Ex
>> start-up times
>> * showed that we reached at most 11 jobs on an Intel i5-6400
>> CPU at 2.70GHz
>> * (a fairly modest desktop CPU). 1 thread was chosen because we
>> don't
>> * really care about getting things to disk quickly just that
>> it's not
>> * blocking other tasks.
>> */
>> util_queue_init(&cache->cache_queue, "disk_cache", 32, 1, 0);
>> + uint8_t cache_version = CACHE_VERSION;
>> + size_t cv_size = sizeof(cache_version);
>> + cache->driver_keys_blob_size = cv_size;
>> +
>> /* Create driver id keys */
>> size_t ts_size = strlen(timestamp) + 1;
>> size_t gpu_name_size = strlen(gpu_name) + 1;
>> - cache->driver_keys_blob_size = ts_size;
>> + cache->driver_keys_blob_size += ts_size;
>> cache->driver_keys_blob_size += gpu_name_size;
>> /* We sometimes store entire structs that contains a pointers in
>> the cache,
>> * use pointer size as a key to avoid hard to debug issues.
>> */
>> uint8_t ptr_size = sizeof(void *);
>> size_t ptr_size_size = sizeof(ptr_size);
>> cache->driver_keys_blob_size += ptr_size_size;
>> size_t driver_flags_size = sizeof(driver_flags);
>> cache->driver_keys_blob_size += driver_flags_size;
>> cache->driver_keys_blob =
>> ralloc_size(cache, cache->driver_keys_blob_size);
>> if (!cache->driver_keys_blob)
>> goto fail;
>> - memcpy(cache->driver_keys_blob, timestamp, ts_size);
>> - memcpy(cache->driver_keys_blob + ts_size, gpu_name, gpu_name_size);
>> - memcpy(cache->driver_keys_blob + ts_size + gpu_name_size, &ptr_size,
>> - ptr_size_size);
>> - memcpy(cache->driver_keys_blob + ts_size + gpu_name_size +
>> ptr_size_size,
>> + memcpy(cache->driver_keys_blob, &cache_version, cv_size);
>> + memcpy(cache->driver_keys_blob + cv_size, timestamp, ts_size);
>> + memcpy(cache->driver_keys_blob + cv_size + ts_size, gpu_name,
>> + gpu_name_size);
>> + memcpy(cache->driver_keys_blob + cv_size + ts_size + gpu_name_size,
>> + &ptr_size, ptr_size_size);
>> + memcpy(cache->driver_keys_blob + cv_size + ts_size +
>> + gpu_name_size + ptr_size_size,
>> &driver_flags, driver_flags_size);
>> /* Seed our rand function */
>> s_rand_xorshift128plus(cache->seed_xorshift128plus, true);
>> ralloc_free(local);
>> return cache;
>> fail:
>>
>
>
More information about the mesa-dev
mailing list