Mesa (master): disk_cache: move cache item loading code into disk_cache_load_item() helper
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Fri Sep 11 06:17:26 UTC 2020
Module: Mesa
Branch: master
Commit: 367ac07efcc80652e3cbec7f02a2db4e5e2084c1
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=367ac07efcc80652e3cbec7f02a2db4e5e2084c1
Author: Timothy Arceri <tarceri at itsqueeze.com>
Date: Thu Aug 6 10:24:21 2020 +1000
disk_cache: move cache item loading code into disk_cache_load_item() helper
This should be helpful if someone chooses to implement cache support on
windows. Also providing this greater level of abstraction makes it easier
to implement alterative cache layouts in future.
Reviewed-by: Bas Nieuwenhuizen <bas at basnieuwenhuizen.nl>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6197>
---
src/util/disk_cache.c | 172 +---------------------------------------------
src/util/disk_cache_os.c | 174 ++++++++++++++++++++++++++++++++++++++++++++++-
src/util/disk_cache_os.h | 3 +
3 files changed, 179 insertions(+), 170 deletions(-)
diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
index c2d5db4f588..91ef18c4e42 100644
--- a/src/util/disk_cache.c
+++ b/src/util/disk_cache.c
@@ -36,11 +36,6 @@
#include <errno.h>
#include <dirent.h>
#include <inttypes.h>
-#include "zlib.h"
-
-#ifdef HAVE_ZSTD
-#include "zstd.h"
-#endif
#include "util/crc32.h"
#include "util/debug.h"
@@ -235,21 +230,6 @@ disk_cache_remove(struct disk_cache *cache, const cache_key key)
disk_cache_evict_item(cache, filename);
}
-static ssize_t
-read_all(int fd, void *buf, size_t count)
-{
- char *in = buf;
- ssize_t read_ret;
- size_t done;
-
- for (done = 0; done < count; done += read_ret) {
- read_ret = read(fd, in + done, count - done);
- if (read_ret == -1 || read_ret == 0)
- return -1;
- }
- return done;
-}
-
static struct disk_cache_put_job *
create_put_job(struct disk_cache *cache, const cache_key key,
const void *data, size_t size,
@@ -362,60 +342,9 @@ disk_cache_put(struct disk_cache *cache, const cache_key key,
}
}
-/**
- * Decompresses cache entry, returns true if successful.
- */
-static bool
-inflate_cache_data(uint8_t *in_data, size_t in_data_size,
- uint8_t *out_data, size_t out_data_size)
-{
-#ifdef HAVE_ZSTD
- size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
- return !ZSTD_isError(ret);
-#else
- z_stream strm;
-
- /* allocate inflate state */
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.next_in = in_data;
- strm.avail_in = in_data_size;
- strm.next_out = out_data;
- strm.avail_out = out_data_size;
-
- int ret = inflateInit(&strm);
- if (ret != Z_OK)
- return false;
-
- ret = inflate(&strm, Z_NO_FLUSH);
- assert(ret != Z_STREAM_ERROR); /* state not clobbered */
-
- /* Unless there was an error we should have decompressed everything in one
- * go as we know the uncompressed file size.
- */
- if (ret != Z_STREAM_END) {
- (void)inflateEnd(&strm);
- return false;
- }
- assert(strm.avail_out == 0);
-
- /* clean up and return */
- (void)inflateEnd(&strm);
- return true;
-#endif
-}
-
void *
disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
{
- int fd = -1, ret;
- struct stat sb;
- char *filename = NULL;
- uint8_t *data = NULL;
- uint8_t *uncompressed_data = NULL;
- uint8_t *file_header = NULL;
-
if (size)
*size = 0;
@@ -441,110 +370,15 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
return blob;
}
- filename = disk_cache_get_cache_filename(cache, key);
+ char *filename = disk_cache_get_cache_filename(cache, key);
if (filename == NULL)
goto fail;
- fd = open(filename, O_RDONLY | O_CLOEXEC);
- if (fd == -1)
- goto fail;
-
- if (fstat(fd, &sb) == -1)
- goto fail;
-
- data = malloc(sb.st_size);
- if (data == NULL)
- goto fail;
-
- size_t ck_size = cache->driver_keys_blob_size;
- file_header = malloc(ck_size);
- if (!file_header)
- goto fail;
-
- if (sb.st_size < ck_size)
- goto fail;
-
- ret = read_all(fd, file_header, ck_size);
- if (ret == -1)
- goto fail;
-
- /* Check for extremely unlikely hash collisions */
- if (memcmp(cache->driver_keys_blob, file_header, ck_size) != 0) {
- assert(!"Mesa cache keys mismatch!");
- goto fail;
- }
-
- size_t cache_item_md_size = sizeof(uint32_t);
- uint32_t md_type;
- ret = read_all(fd, &md_type, cache_item_md_size);
- if (ret == -1)
- goto fail;
-
- if (md_type == CACHE_ITEM_TYPE_GLSL) {
- uint32_t num_keys;
- cache_item_md_size += sizeof(uint32_t);
- ret = read_all(fd, &num_keys, sizeof(uint32_t));
- if (ret == -1)
- goto fail;
-
- /* The cache item metadata is currently just used for distributing
- * precompiled shaders, they are not used by Mesa so just skip them for
- * now.
- * TODO: pass the metadata back to the caller and do some basic
- * validation.
- */
- cache_item_md_size += num_keys * sizeof(cache_key);
- ret = lseek(fd, num_keys * sizeof(cache_key), SEEK_CUR);
- if (ret == -1)
- goto fail;
- }
-
- /* Load the CRC that was created when the file was written. */
- struct cache_entry_file_data cf_data;
- size_t cf_data_size = sizeof(cf_data);
- ret = read_all(fd, &cf_data, cf_data_size);
- if (ret == -1)
- goto fail;
+ return disk_cache_load_item(cache, filename, size);
- /* Load the actual cache data. */
- size_t cache_data_size =
- sb.st_size - cf_data_size - ck_size - cache_item_md_size;
- ret = read_all(fd, data, cache_data_size);
- if (ret == -1)
- goto fail;
-
- /* Uncompress the cache data */
- uncompressed_data = malloc(cf_data.uncompressed_size);
- if (!inflate_cache_data(data, cache_data_size, uncompressed_data,
- cf_data.uncompressed_size))
- goto fail;
-
- /* Check the data for corruption */
- if (cf_data.crc32 != util_hash_crc32(uncompressed_data,
- cf_data.uncompressed_size))
- goto fail;
-
- free(data);
- free(filename);
- free(file_header);
- close(fd);
-
- if (size)
- *size = cf_data.uncompressed_size;
-
- return uncompressed_data;
-
- fail:
- if (data)
- free(data);
- if (uncompressed_data)
- free(uncompressed_data);
+fail:
if (filename)
free(filename);
- if (file_header)
- free(file_header);
- if (fd != -1)
- close(fd);
return NULL;
}
diff --git a/src/util/disk_cache_os.c b/src/util/disk_cache_os.c
index d23b2f7d82f..fd49bff6dec 100644
--- a/src/util/disk_cache_os.c
+++ b/src/util/disk_cache_os.c
@@ -24,6 +24,8 @@
#ifdef ENABLE_SHADER_CACHE
#include <assert.h>
+#include <inttypes.h>
+#include <stdbool.h>
#include <stddef.h>
#include <stdlib.h>
@@ -137,6 +139,50 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
# endif
}
+/**
+ * Decompresses cache entry, returns true if successful.
+ */
+static bool
+inflate_cache_data(uint8_t *in_data, size_t in_data_size,
+ uint8_t *out_data, size_t out_data_size)
+{
+#ifdef HAVE_ZSTD
+ size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
+ return !ZSTD_isError(ret);
+#else
+ z_stream strm;
+
+ /* allocate inflate state */
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.next_in = in_data;
+ strm.avail_in = in_data_size;
+ strm.next_out = out_data;
+ strm.avail_out = out_data_size;
+
+ int ret = inflateInit(&strm);
+ if (ret != Z_OK)
+ return false;
+
+ ret = inflate(&strm, Z_NO_FLUSH);
+ assert(ret != Z_STREAM_ERROR); /* state not clobbered */
+
+ /* Unless there was an error we should have decompressed everything in one
+ * go as we know the uncompressed file size.
+ */
+ if (ret != Z_STREAM_END) {
+ (void)inflateEnd(&strm);
+ return false;
+ }
+ assert(strm.avail_out == 0);
+
+ /* clean up and return */
+ (void)inflateEnd(&strm);
+ return true;
+#endif
+}
+
#if DETECT_OS_WINDOWS
/* TODO: implement disk cache support on windows */
@@ -144,7 +190,6 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
#include <dirent.h>
#include <errno.h>
-#include <inttypes.h>
#include <pwd.h>
#include <stdio.h>
#include <string.h>
@@ -154,6 +199,7 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
#include <sys/stat.h>
#include <unistd.h>
+#include "util/crc32.h"
#include "util/debug.h"
#include "util/disk_cache.h"
#include "util/disk_cache_os.h"
@@ -378,6 +424,21 @@ make_cache_file_directory(struct disk_cache *cache, const cache_key key)
free(dir);
}
+static ssize_t
+read_all(int fd, void *buf, size_t count)
+{
+ char *in = buf;
+ ssize_t read_ret;
+ size_t done;
+
+ for (done = 0; done < count; done += read_ret) {
+ read_ret = read(fd, in + done, count - done);
+ if (read_ret == -1 || read_ret == 0)
+ return -1;
+ }
+ return done;
+}
+
static ssize_t
write_all(int fd, const void *buf, size_t count)
{
@@ -454,6 +515,117 @@ disk_cache_evict_item(struct disk_cache *cache, char *filename)
p_atomic_add(cache->size, - (uint64_t)sb.st_blocks * 512);
}
+void *
+disk_cache_load_item(struct disk_cache *cache, char *filename, size_t *size)
+{
+ int fd = -1, ret;
+ struct stat sb;
+ uint8_t *data = NULL;
+ uint8_t *uncompressed_data = NULL;
+ uint8_t *file_header = NULL;
+
+ fd = open(filename, O_RDONLY | O_CLOEXEC);
+ if (fd == -1)
+ goto fail;
+
+ if (fstat(fd, &sb) == -1)
+ goto fail;
+
+ data = malloc(sb.st_size);
+ if (data == NULL)
+ goto fail;
+
+ size_t ck_size = cache->driver_keys_blob_size;
+ file_header = malloc(ck_size);
+ if (!file_header)
+ goto fail;
+
+ if (sb.st_size < ck_size)
+ goto fail;
+
+ ret = read_all(fd, file_header, ck_size);
+ if (ret == -1)
+ goto fail;
+
+ /* Check for extremely unlikely hash collisions */
+ if (memcmp(cache->driver_keys_blob, file_header, ck_size) != 0) {
+ assert(!"Mesa cache keys mismatch!");
+ goto fail;
+ }
+
+ size_t cache_item_md_size = sizeof(uint32_t);
+ uint32_t md_type;
+ ret = read_all(fd, &md_type, cache_item_md_size);
+ if (ret == -1)
+ goto fail;
+
+ if (md_type == CACHE_ITEM_TYPE_GLSL) {
+ uint32_t num_keys;
+ cache_item_md_size += sizeof(uint32_t);
+ ret = read_all(fd, &num_keys, sizeof(uint32_t));
+ if (ret == -1)
+ goto fail;
+
+ /* The cache item metadata is currently just used for distributing
+ * precompiled shaders, they are not used by Mesa so just skip them for
+ * now.
+ * TODO: pass the metadata back to the caller and do some basic
+ * validation.
+ */
+ cache_item_md_size += num_keys * sizeof(cache_key);
+ ret = lseek(fd, num_keys * sizeof(cache_key), SEEK_CUR);
+ if (ret == -1)
+ goto fail;
+ }
+
+ /* Load the CRC that was created when the file was written. */
+ struct cache_entry_file_data cf_data;
+ size_t cf_data_size = sizeof(cf_data);
+ ret = read_all(fd, &cf_data, cf_data_size);
+ if (ret == -1)
+ goto fail;
+
+ /* Load the actual cache data. */
+ size_t cache_data_size =
+ sb.st_size - cf_data_size - ck_size - cache_item_md_size;
+ ret = read_all(fd, data, cache_data_size);
+ if (ret == -1)
+ goto fail;
+
+ /* Uncompress the cache data */
+ uncompressed_data = malloc(cf_data.uncompressed_size);
+ if (!inflate_cache_data(data, cache_data_size, uncompressed_data,
+ cf_data.uncompressed_size))
+ goto fail;
+
+ /* Check the data for corruption */
+ if (cf_data.crc32 != util_hash_crc32(uncompressed_data,
+ cf_data.uncompressed_size))
+ goto fail;
+
+ free(data);
+ free(filename);
+ free(file_header);
+ close(fd);
+
+ if (size)
+ *size = cf_data.uncompressed_size;
+
+ return uncompressed_data;
+
+ fail:
+ if (data)
+ free(data);
+ if (uncompressed_data)
+ free(uncompressed_data);
+ if (file_header)
+ free(file_header);
+ if (fd != -1)
+ close(fd);
+
+ return NULL;
+}
+
/* Return a filename within the cache's directory corresponding to 'key'. The
* returned filename is ralloced with 'cache' as the parent context.
*
diff --git a/src/util/disk_cache_os.h b/src/util/disk_cache_os.h
index e753d08bd99..490e8078cec 100644
--- a/src/util/disk_cache_os.h
+++ b/src/util/disk_cache_os.h
@@ -103,6 +103,9 @@ disk_cache_evict_lru_item(struct disk_cache *cache);
void
disk_cache_evict_item(struct disk_cache *cache, char *filename);
+void *
+disk_cache_load_item(struct disk_cache *cache, char *filename, size_t *size);
+
char *
disk_cache_get_cache_filename(struct disk_cache *cache, const cache_key key);
More information about the mesa-commit
mailing list