Mesa (master): util: Use ZSTD for shader cache if possible
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Mon Nov 11 18:54:16 UTC 2019
Module: Mesa
Branch: master
Commit: a8d941091f72923561a6c58b46ccb264b6a0e205
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=a8d941091f72923561a6c58b46ccb264b6a0e205
Author: Dylan Baker <dylan at pnwbakers.com>
Date: Thu Oct 31 13:26:00 2019 -0700
util: Use ZSTD for shader cache if possible
This allows ZSTD instead of ZLIB to be used for compressing the shader
cache.
On a 72 core system emulating skl with a full shader-db (with i965):
ZSTD:
1915.10s user 229.27s system 5150% cpu 41.632 total (cold cache)
225.40s user 10.87s system 3810% cpu 6.201 total (warm cache)
154M (235M on disk)
ZLIB:
2231.33s user 194.24s system 1899% cpu 2:07.72 total (cold cache)
229.15s user 10.63s system 3906% cpu 6.139 total (warm cache)
163M (244M on disk)
Tim Arceri sees (8 core ryzen and a full shader-db):
ZSTD:
2505.22 user 40.50 system 3:18.73 elapsed 1280% CPU (cold cache)
418.71 user 14.93 system 0:46.53 elapsed 931% CPU (warm cache)
454.3 MB (681.7 MB on disk)
ZLIB:
3069.83 user 40.02 system 4:20.13 elapsed 1195% CPU (cold cache)
425.50 user 15.17 system 0:46.80 elapsed 941% CPU (warm cache)
470.3 MB (701.4 MB on disk)
Reviewed-by: Eric Engestrom <eric.engestrom at intel.com> (v1)
Reviewed-by: Eric Anholt <eric at anholt.net>
---
meson.build | 11 +++++++++++
meson_options.txt | 9 ++++++++-
src/util/disk_cache.c | 30 ++++++++++++++++++++++++++++++
src/util/meson.build | 1 +
4 files changed, 50 insertions(+), 1 deletion(-)
diff --git a/meson.build b/meson.build
index b9a6da01d31..39907709f71 100644
--- a/meson.build
+++ b/meson.build
@@ -1251,6 +1251,17 @@ endif
# TODO: some of these may be conditional
dep_zlib = dependency('zlib', version : '>= 1.2.3', fallback : ['zlib', 'zlib_dep'])
pre_args += '-DHAVE_ZLIB'
+
+_zstd = get_option('zstd')
+if _zstd != 'false'
+ dep_zstd = dependency('libzstd', required : _zstd == 'true')
+ if dep_zstd.found()
+ pre_args += '-DHAVE_ZSTD'
+ endif
+else
+ dep_zstd = null_dep
+endif
+
dep_thread = dependency('threads')
if dep_thread.found() and host_machine.system() != 'windows'
pre_args += '-DHAVE_PTHREAD'
diff --git a/meson_options.txt b/meson_options.txt
index 626baf3d5c2..8c3e9978529 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -1,4 +1,4 @@
-# Copyright © 2017-2018 Intel Corporation
+# Copyright © 2017-2019 Intel Corporation
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -366,3 +366,10 @@ option(
value : 25,
description : 'Android Platform SDK version. Default: Nougat version.'
)
+option(
+ 'zstd',
+ type : 'combo',
+ choices : ['auto', 'true', 'false'],
+ value : 'auto',
+ description : 'Use ZSTD instead of ZLIB in some cases.'
+)
diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
index 0cd92ca2d5b..e5b21b9a79f 100644
--- a/src/util/disk_cache.c
+++ b/src/util/disk_cache.c
@@ -40,6 +40,10 @@
#include <inttypes.h>
#include "zlib.h"
+#ifdef HAVE_ZSTD
+#include "zstd.h"
+#endif
+
#include "util/crc32.h"
#include "util/debug.h"
#include "util/rand_xor.h"
@@ -75,6 +79,9 @@
*/
#define CACHE_VERSION 1
+/* 3 is the recomended level, with 22 as the absolute maximum */
+#define ZSTD_COMPRESSION_LEVEL 3
+
struct disk_cache {
/* The path to the cache directory. */
char *path;
@@ -738,6 +745,23 @@ static size_t
deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
const char *filename)
{
+#ifdef HAVE_ZSTD
+ /* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
+ * compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+ */
+ size_t out_size = ZSTD_compressBound(in_data_size);
+ void * out = malloc(out_size);
+
+ size_t ret = ZSTD_compress(out, out_size, in_data, in_data_size,
+ ZSTD_COMPRESSION_LEVEL);
+ if (ZSTD_isError(ret)) {
+ free(out);
+ return 0;
+ }
+ write_all(dest, out, ret);
+ free(out);
+ return ret;
+#else
unsigned char *out;
/* allocate deflate state */
@@ -798,6 +822,7 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
(void)deflateEnd(&strm);
free(out);
return compressed_size;
+# endif
}
static struct disk_cache_put_job *
@@ -1059,6 +1084,10 @@ static bool
inflate_cache_data(uint8_t *in_data, size_t in_data_size,
uint8_t *out_data, size_t out_data_size)
{
+#ifdef HAVE_ZSTD
+ size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
+ return !ZSTD_isError(ret);
+#else
z_stream strm;
/* allocate inflate state */
@@ -1089,6 +1118,7 @@ inflate_cache_data(uint8_t *in_data, size_t in_data_size,
/* clean up and return */
(void)inflateEnd(&strm);
return true;
+#endif
}
void *
diff --git a/src/util/meson.build b/src/util/meson.build
index 972d757195a..a7990ad52cc 100644
--- a/src/util/meson.build
+++ b/src/util/meson.build
@@ -149,6 +149,7 @@ deps_for_libmesa_util = [
dep_atomic,
dep_m,
dep_valgrind,
+ dep_zstd,
]
if with_platform_android
More information about the mesa-commit
mailing list