Mesa (master): util/disk_cache: make use of the new compression helpers
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Mon Mar 15 04:02:48 UTC 2021
Module: Mesa
Branch: master
Commit: 4a98764da8d12ce2190ff14400c5824f2de93b87
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=4a98764da8d12ce2190ff14400c5824f2de93b87
Author: Timothy Arceri <tarceri at itsqueeze.com>
Date: Thu Mar 4 16:39:47 2021 +1100
util/disk_cache: make use of the new compression helpers
This makes compression use more consistent between the zstd and
zlib libraries. It also reduces the amount of code required for
zlib use.
Reviewed-By: Mike Blumenkrantz <michael.blumenkrantz at gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9593>
---
src/util/disk_cache_os.c | 233 ++++++++---------------------------------------
1 file changed, 38 insertions(+), 195 deletions(-)
diff --git a/src/util/disk_cache_os.c b/src/util/disk_cache_os.c
index 1f6cbeb3bd4..c4a0324725a 100644
--- a/src/util/disk_cache_os.c
+++ b/src/util/disk_cache_os.c
@@ -33,12 +33,7 @@
#include <dirent.h>
#include <fcntl.h>
-#include "zlib.h"
-
-#ifdef HAVE_ZSTD
-#include "zstd.h"
-#endif
-
+#include "util/compress.h"
#include "util/crc32.h"
struct cache_entry_file_data {
@@ -46,181 +41,6 @@ struct cache_entry_file_data {
uint32_t uncompressed_size;
};
-/* 3 is the recomended level, with 22 as the absolute maximum */
-#define ZSTD_COMPRESSION_LEVEL 3
-
-/* From the zlib docs:
- * "If the memory is available, buffers sizes on the order of 128K or 256K
- * bytes should be used."
- */
-#define BUFSIZE 256 * 1024
-
-static ssize_t
-write_all(int fd, const void *buf, size_t count);
-
-/**
- * Compresses cache entry in memory and writes it to disk. Returns the size
- * of the data written to disk.
- */
-static size_t
-deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest)
-{
-#ifdef HAVE_ZSTD
- /* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
- * compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
- */
- size_t out_size = ZSTD_compressBound(in_data_size);
- void * out = malloc(out_size);
-
- size_t ret = ZSTD_compress(out, out_size, in_data, in_data_size,
- ZSTD_COMPRESSION_LEVEL);
- if (ZSTD_isError(ret)) {
- free(out);
- return 0;
- }
-
- /* Create CRC of the compressed data. We will read this when restoring the
- * cache and use it to check for corruption.
- */
- struct cache_entry_file_data cf_data;
- cf_data.crc32 = util_hash_crc32(out, ret);
- cf_data.uncompressed_size = in_data_size;
-
- size_t cf_data_size = sizeof(cf_data);
- ssize_t written = write_all(dest, &cf_data, cf_data_size);
- if (written == -1) {
- free(out);
- return 0;
- }
-
- written = write_all(dest, out, ret);
- if (written == -1) {
- free(out);
- return 0;
- }
- free(out);
- return ret;
-#else
- /* Create CRC of the uncompressed data. We will read this when restoring
- * the cache and use it to check for corruption.
- */
- struct cache_entry_file_data cf_data;
- cf_data.crc32 = util_hash_crc32(in_data, in_data_size);
- cf_data.uncompressed_size = in_data_size;
-
- size_t cf_data_size = sizeof(cf_data);
- ssize_t written = write_all(dest, &cf_data, cf_data_size);
- if (written == -1) {
- return 0;
- }
-
- unsigned char *out;
-
- /* allocate deflate state */
- z_stream strm;
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.next_in = (uint8_t *) in_data;
- strm.avail_in = in_data_size;
-
- int ret = deflateInit(&strm, Z_BEST_COMPRESSION);
- if (ret != Z_OK)
- return 0;
-
- /* compress until end of in_data */
- size_t compressed_size = 0;
- int flush;
-
- out = malloc(BUFSIZE * sizeof(unsigned char));
- if (out == NULL)
- return 0;
-
- do {
- int remaining = in_data_size - BUFSIZE;
- flush = remaining > 0 ? Z_NO_FLUSH : Z_FINISH;
- in_data_size -= BUFSIZE;
-
- /* Run deflate() on input until the output buffer is not full (which
- * means there is no more data to deflate).
- */
- do {
- strm.avail_out = BUFSIZE;
- strm.next_out = out;
-
- ret = deflate(&strm, flush); /* no bad return value */
- assert(ret != Z_STREAM_ERROR); /* state not clobbered */
-
- size_t have = BUFSIZE - strm.avail_out;
- compressed_size += have;
-
- written = write_all(dest, out, have);
- if (written == -1) {
- (void)deflateEnd(&strm);
- free(out);
- return 0;
- }
- } while (strm.avail_out == 0);
-
- /* all input should be used */
- assert(strm.avail_in == 0);
-
- } while (flush != Z_FINISH);
-
- /* stream should be complete */
- assert(ret == Z_STREAM_END);
-
- /* clean up and return */
- (void)deflateEnd(&strm);
- free(out);
- return compressed_size;
-# endif
-}
-
-/**
- * Decompresses cache entry, returns true if successful.
- */
-static bool
-inflate_cache_data(uint8_t *in_data, size_t in_data_size,
- uint8_t *out_data, size_t out_data_size)
-{
-#ifdef HAVE_ZSTD
- size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
- return !ZSTD_isError(ret);
-#else
- z_stream strm;
-
- /* allocate inflate state */
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.next_in = in_data;
- strm.avail_in = in_data_size;
- strm.next_out = out_data;
- strm.avail_out = out_data_size;
-
- int ret = inflateInit(&strm);
- if (ret != Z_OK)
- return false;
-
- ret = inflate(&strm, Z_NO_FLUSH);
- assert(ret != Z_STREAM_ERROR); /* state not clobbered */
-
- /* Unless there was an error we should have decompressed everything in one
- * go as we know the uncompressed file size.
- */
- if (ret != Z_STREAM_END) {
- (void)inflateEnd(&strm);
- return false;
- }
- assert(strm.avail_out == 0);
-
- /* clean up and return */
- (void)inflateEnd(&strm);
- return true;
-#endif
-}
-
#if DETECT_OS_WINDOWS
/* TODO: implement disk cache support on windows */
@@ -630,25 +450,16 @@ disk_cache_load_item(struct disk_cache *cache, char *filename, size_t *size)
if (ret == -1)
goto fail;
-#ifdef HAVE_ZSTD
/* Check the data for corruption */
if (cf_data.crc32 != util_hash_crc32(data, cache_data_size))
goto fail;
-#endif
/* Uncompress the cache data */
uncompressed_data = malloc(cf_data.uncompressed_size);
- if (!inflate_cache_data(data, cache_data_size, uncompressed_data,
- cf_data.uncompressed_size))
+ if (!util_compress_inflate(data, cache_data_size, uncompressed_data,
+ cf_data.uncompressed_size))
goto fail;
-#ifndef HAVE_ZSTD
- /* Check the data for corruption */
- if (cf_data.crc32 != util_hash_crc32(uncompressed_data,
- cf_data.uncompressed_size))
- goto fail;
-#endif
-
free(data);
free(filename);
free(file_header);
@@ -700,6 +511,7 @@ disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job,
char *filename)
{
int fd = -1, fd_final = -1;
+ void *compressed_data = NULL;
/* Write to a temporary file to allow for an atomic rename to the
* final destination filename, (to prevent any readers from seeing
@@ -798,16 +610,46 @@ disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job,
}
}
+ /* Compress the cache item data */
+ size_t max_buf = util_compress_max_compressed_len(dc_job->size);
+ compressed_data = malloc(max_buf);
+ if (compressed_data == NULL) {
+ unlink(filename_tmp);
+ goto done;
+ }
+
+ size_t compressed_size =
+ util_compress_deflate(dc_job->data, dc_job->size,
+ compressed_data, max_buf);
+ if (compressed_size == 0) {
+ unlink(filename_tmp);
+ goto done;
+ }
+
+ /* Create CRC of the compressed data. We will read this when restoring the
+ * cache and use it to check for corruption.
+ */
+ struct cache_entry_file_data cf_data;
+ cf_data.crc32 = util_hash_crc32(compressed_data, compressed_size);
+ cf_data.uncompressed_size = dc_job->size;
+
/* Now, finally, write out the contents to the temporary file, then
* rename them atomically to the destination filename, and also
* perform an atomic increment of the total cache size.
*/
- size_t file_size = deflate_and_write_to_disk(dc_job->data, dc_job->size,
- fd);
- if (file_size == 0) {
+ size_t cf_data_size = sizeof(cf_data);
+ ret = write_all(fd, &cf_data, cf_data_size);
+ if (ret == -1) {
unlink(filename_tmp);
goto done;
}
+
+ ret = write_all(fd, compressed_data, compressed_size);
+ if (ret == -1) {
+ unlink(filename_tmp);
+ goto done;
+ }
+
ret = rename(filename_tmp, filename);
if (ret == -1) {
unlink(filename_tmp);
@@ -832,6 +674,7 @@ disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job,
if (fd != -1)
close(fd);
free(filename_tmp);
+ free(compressed_data);
}
/* Determine path for cache based on the first defined name as follows:
More information about the mesa-commit
mailing list