[Mesa-dev] [PATCH 6/7] util/disk_cache: hash timestamps into the cache keys

[Timothy Arceri]

We should get rid of the gpu id dir too as it will never get cleaned up 
should a user upgrade their gpu or copy their home dir to a new PC.

We also need to write the timestamp and gpu id to the header of each 
cache entry (like we do with the filesize). This will be used for the 
collection and distribution of compiled shaders.

On 13/03/17 05:32, Grazvydas Ignotas wrote:
> Instead of using a directory, hash the timestamps into the cache keys
> themselves. Since there is no more timestamp directory, there is no more
> need for deleting the cache of other mesa versions and we rely on
> eviction to clean up the old cache entries. This solves the problem of
> using several incarnations of disk_cache at the same time, where one
> deletes a directory belonging to the other, like when both OpenGL and
> gallium nine are used simultaneously (or several different mesa
> installations).
>
> Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=100091
> Signed-off-by: Grazvydas Ignotas <notasas at gmail.com>
> ---
>  src/compiler/glsl/tests/cache_test.c | 10 +++----
>  src/util/disk_cache.c                | 56 ++++--------------------------------
>  2 files changed, 11 insertions(+), 55 deletions(-)
>
> diff --git a/src/compiler/glsl/tests/cache_test.c b/src/compiler/glsl/tests/cache_test.c
> index 5c3a1ad..32556c2 100644
> --- a/src/compiler/glsl/tests/cache_test.c
> +++ b/src/compiler/glsl/tests/cache_test.c
> @@ -127,7 +127,7 @@ rmrf_local(const char *path)
>  }
>
>  static void
> -check_timestamp_and_gpu_id_directories_created(char *cache_dir)
> +check_directories_created(char *cache_dir)
>  {
>     bool sub_dirs_created = false;
>
> @@ -183,13 +183,13 @@ test_disk_cache_create(void)
>     /* Create string with expected directory hierarchy */
>     char expected_dir_h[255];
>     sprintf(expected_dir_h, "%s%s%s", CACHE_TEST_TMP "/xdg-cache-home/mesa/",
> -           get_arch_bitness_str(), "/make_check/test");
> +           get_arch_bitness_str(), "/test");
>
>     mkdir(CACHE_TEST_TMP, 0755);
>     cache = disk_cache_create("test", "make_check");
>     expect_non_null(cache, "disk_cache_create with XDG_CACHE_HOME set");
>
> -   check_timestamp_and_gpu_id_directories_created(expected_dir_h);
> +   check_directories_created(expected_dir_h);
>
>     disk_cache_destroy(cache);
>
> @@ -204,13 +204,13 @@ test_disk_cache_create(void)
>
>     sprintf(expected_dir_h, "%s%s%s", CACHE_TEST_TMP
>             "/mesa-glsl-cache-dir/mesa/", get_arch_bitness_str(),
> -           "/make_check/test");
> +           "/test");
>
>     mkdir(CACHE_TEST_TMP, 0755);
>     cache = disk_cache_create("test", "make_check");
>     expect_non_null(cache, "disk_cache_create with MESA_GLSL_CACHE_DIR set");
>
> -   check_timestamp_and_gpu_id_directories_created(expected_dir_h);
> +   check_directories_created(expected_dir_h);
>
>     disk_cache_destroy(cache);
>  }
> diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
> index b639907..3055179 100644
> --- a/src/util/disk_cache.c
> +++ b/src/util/disk_cache.c
> @@ -137,45 +137,8 @@ concatenate_and_mkdir(void *ctx, const char *path, const char *name)
>        return NULL;
>  }
>
> -static int
> -remove_dir(const char *fpath, const struct stat *sb,
> -           int typeflag, struct FTW *ftwbuf)
> -{
> -   if (S_ISREG(sb->st_mode))
> -      unlink(fpath);
> -   else if (S_ISDIR(sb->st_mode))
> -      rmdir(fpath);
> -
> -   return 0;
> -}
> -
> -static void
> -remove_old_cache_directories(void *mem_ctx, const char *path,
> -                             const char *timestamp)
> -{
> -   DIR *dir = opendir(path);
> -
> -   struct dirent* d_entry;
> -   while((d_entry = readdir(dir)) != NULL)
> -   {
> -      char *full_path =
> -         ralloc_asprintf(mem_ctx, "%s/%s", path, d_entry->d_name);
> -
> -      struct stat sb;
> -      if (stat(full_path, &sb) == 0 && S_ISDIR(sb.st_mode) &&
> -          strcmp(d_entry->d_name, timestamp) != 0 &&
> -          strcmp(d_entry->d_name, "..") != 0 &&
> -          strcmp(d_entry->d_name, ".") != 0) {
> -         nftw(full_path, remove_dir, 20, FTW_DEPTH);
> -      }
> -   }
> -
> -   closedir(dir);
> -}
> -
>  static char *
> -create_mesa_cache_dir(void *mem_ctx, const char *path, const char *timestamp,
> -                      const char *gpu_name)
> +create_mesa_cache_dir(void *mem_ctx, const char *path, const char *gpu_name)
>  {
>     char *new_path = concatenate_and_mkdir(mem_ctx, path, "mesa");
>     if (new_path == NULL)
> @@ -190,13 +153,6 @@ create_mesa_cache_dir(void *mem_ctx, const char *path, const char *timestamp,
>     if (new_path == NULL)
>        return NULL;
>
> -   /* Remove cache directories for old Mesa versions */
> -   remove_old_cache_directories(mem_ctx, new_path, timestamp);
> -
> -   new_path = concatenate_and_mkdir(mem_ctx, new_path, timestamp);
> -   if (new_path == NULL)
> -      return NULL;
> -
>     new_path = concatenate_and_mkdir(mem_ctx, new_path, gpu_name);
>     if (new_path == NULL)
>        return NULL;
> @@ -239,8 +195,7 @@ disk_cache_create(const char *gpu_name, const char *timestamp)
>        if (mkdir_if_needed(path) == -1)
>           goto fail;
>
> -      path = create_mesa_cache_dir(local, path, timestamp,
> -                                   gpu_name);
> +      path = create_mesa_cache_dir(local, path, gpu_name);
>        if (path == NULL)
>           goto fail;
>     }
> @@ -252,8 +207,7 @@ disk_cache_create(const char *gpu_name, const char *timestamp)
>           if (mkdir_if_needed(xdg_cache_home) == -1)
>              goto fail;
>
> -         path = create_mesa_cache_dir(local, xdg_cache_home, timestamp,
> -                                      gpu_name);
> +         path = create_mesa_cache_dir(local, xdg_cache_home, gpu_name);
>           if (path == NULL)
>              goto fail;
>        }
> @@ -289,7 +243,7 @@ disk_cache_create(const char *gpu_name, const char *timestamp)
>        if (path == NULL)
>           goto fail;
>
> -      path = create_mesa_cache_dir(local, path, timestamp, gpu_name);
> +      path = create_mesa_cache_dir(local, path, gpu_name);
>        if (path == NULL)
>           goto fail;
>     }
> @@ -302,6 +256,8 @@ disk_cache_create(const char *gpu_name, const char *timestamp)
>     if (cache->key_hash_base == NULL)
>        goto fail;
>
> +   _mesa_sha1_update(cache->key_hash_base, timestamp, strlen(timestamp));
> +
>     cache->path = ralloc_strdup(cache, path);
>     if (cache->path == NULL)
>        goto fail;
>