[systemd-devel] [PATCH 1/2] journal: add LZ4 as optional compressor
Zbigniew Jędrzejewski-Szmek
zbyszek at in.waw.pl
Sat Jul 5 11:56:55 PDT 2014
Add liblz4 as an optional dependency when requested with --enable-lz4, and
use it in preference to liblzma for journal blob and coredump compression. To
retain backwards compatibility, XZ is used to decompress old blobs.
Things will function correctly only with lz4-119.
Based on the benchmarks found on the web, lz4 seems to be the best choice for
"quick" compressors atm. If we decide to go this way, I'll work with upstream
to improve their packaging practices, and at least use pkg-config.
---
Makefile.am | 11 +-
configure.ac | 15 +-
src/journal/compress.c | 382 +++++++++++++++++++++++++++++++++++++++----
src/journal/compress.h | 65 +++++++-
src/journal/coredump.c | 10 +-
src/journal/coredumpctl.c | 2 +-
src/journal/journal-def.h | 28 +++-
src/journal/journal-file.c | 99 ++++++-----
src/journal/journal-file.h | 10 +-
src/journal/journal-verify.c | 51 ++++--
src/journal/sd-journal.c | 36 ++--
src/journal/test-compress.c | 158 +++++++++++++-----
12 files changed, 688 insertions(+), 179 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 55a7546eef..c85d66ef32 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3526,14 +3526,12 @@ test_catalog_CPPFLAGS = \
test_catalog_LDADD = \
libsystemd-journal-core.la
-if HAVE_XZ
test_compress_SOURCES = \
src/journal/test-compress.c
test_compress_LDADD = \
libsystemd-journal-internal.la \
libsystemd-shared.la
-endif
libsystemd_journal_core_la_SOURCES = \
src/journal/journald-kmsg.c \
@@ -3617,9 +3615,7 @@ tests += \
test-mmap-cache \
test-catalog
-if HAVE_XZ
tests += test-compress
-endif
pkginclude_HEADERS += \
src/systemd/sd-journal.h \
@@ -3652,10 +3648,10 @@ libsystemd_journal_internal_la_CFLAGS = \
libsystemd_journal_internal_la_LIBADD =
-if HAVE_XZ
libsystemd_journal_internal_la_SOURCES += \
src/journal/compress.c
+if HAVE_XZ
libsystemd_journal_internal_la_CFLAGS += \
$(XZ_CFLAGS)
@@ -3663,6 +3659,11 @@ libsystemd_journal_internal_la_LIBADD += \
$(XZ_LIBS)
endif
+if HAVE_LZ4
+libsystemd_journal_internal_la_LIBADD += \
+ -llz4
+endif
+
if HAVE_GCRYPT
libsystemd_journal_internal_la_SOURCES += \
src/journal/journal-authenticate.c \
diff --git a/configure.ac b/configure.ac
index 93aba06739..7a04ca4fa0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -503,14 +503,24 @@ have_xz=no
AC_ARG_ENABLE(xz, AS_HELP_STRING([--disable-xz], [Disable optional XZ support]))
if test "x$enable_xz" != "xno"; then
PKG_CHECK_MODULES(XZ, [ liblzma ],
- [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes], have_xz=no)
+ [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes])
if test "x$have_xz" = xno -a "x$enable_xz" = xyes; then
- AC_MSG_ERROR([*** Xz support requested but libraries not found])
+ AC_MSG_ERROR([*** XZ support requested but libraries not found])
fi
fi
AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
# ------------------------------------------------------------------------------
+have_lz4=no
+AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
+AS_IF([test "x$enable_lz4" == "xyes"], [
+ AC_CHECK_HEADERS(lz4.h,
+ [AC_DEFINE(HAVE_LZ4, 1, [Define in LZ4 is available]) have_lz4=yes],
+ [AC_MSG_ERROR([*** LZ4 support requested but headers not found])])
+])
+AM_CONDITIONAL(HAVE_LZ4, [test "$have_lz4" = "yes"])
+
+# ------------------------------------------------------------------------------
AC_ARG_ENABLE([pam],
AS_HELP_STRING([--disable-pam],[Disable optional PAM support]),
[case "${enableval}" in
@@ -1266,6 +1276,7 @@ AC_MSG_RESULT([
SECCOMP: ${have_seccomp}
SMACK: ${have_smack}
XZ: ${have_xz}
+ LZ4: ${have_lz4}
ACL: ${have_acl}
GCRYPT: ${have_gcrypt}
QRENCODE: ${have_qrencode}
diff --git a/src/journal/compress.c b/src/journal/compress.c
index 37c55a8728..f7fc665e9c 100644
--- a/src/journal/compress.c
+++ b/src/journal/compress.c
@@ -23,13 +23,30 @@
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
-#include <lzma.h>
+
+#ifdef HAVE_XZ
+# include <lzma.h>
+#endif
+
+#ifdef HAVE_LZ4
+# include <lz4.h>
+#endif
#include "compress.h"
#include "macro.h"
#include "util.h"
+#include "sparse-endian.h"
+#include "journal-def.h"
+
+static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
+ [OBJECT_COMPRESSED_XZ] = "XZ",
+ [OBJECT_COMPRESSED_LZ4] = "LZ4",
+};
+
+DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
-bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+#ifdef HAVE_XZ
lzma_ret ret;
size_t out_pos = 0;
@@ -42,21 +59,50 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_
* compressed result is longer than the original */
ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
- src, src_size, dst, &out_pos, src_size);
+ src, src_size, dst, &out_pos, src_size - 1);
if (ret != LZMA_OK)
- return false;
-
- /* Is it actually shorter? */
- if (out_pos == src_size)
- return false;
+ return -ENOSPC;
*dst_size = out_pos;
- return true;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
}
-bool uncompress_blob(const void *src, uint64_t src_size,
- void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+#ifdef HAVE_LZ4
+ int r;
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_size);
+
+ /* Returns false if we couldn't compress the data or the
+ * compressed result is longer than the original */
+
+ if (src_size < 9)
+ return -ENOSPC;
+
+ r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
+ if (r <= 0)
+ return -ENOSPC;
+
+ *(le64_t*) dst = htole64(src_size);
+ *dst_size = r + 8;
+
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+
+int decompress_blob_xz(const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+
+#ifdef HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
uint64_t space;
@@ -70,7 +116,7 @@ bool uncompress_blob(const void *src, uint64_t src_size,
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
if (ret != LZMA_OK)
- return false;
+ return -ENOMEM;
space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
if (!greedy_realloc(dst, dst_alloc_size, space, 1))
@@ -89,15 +135,13 @@ bool uncompress_blob(const void *src, uint64_t src_size,
if (ret == LZMA_STREAM_END)
break;
-
- if (ret != LZMA_OK)
- return false;
+ else if (ret != LZMA_OK)
+ return -ENOMEM;
if (dst_max > 0 && (space - s.avail_out) >= dst_max)
break;
-
- if (dst_max > 0 && space == dst_max)
- return false;
+ else if (dst_max > 0 && space == dst_max)
+ return -E2BIG;
used = space - s.avail_out;
space = MIN(2 * space, dst_max ?: (uint64_t) -1);
@@ -109,18 +153,75 @@ bool uncompress_blob(const void *src, uint64_t src_size,
}
*dst_size = space - s.avail_out;
- return true;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
}
-bool uncompress_startswith(const void *src, uint64_t src_size,
- void **buffer, uint64_t *buffer_size,
- const void *prefix, uint64_t prefix_len,
- uint8_t extra) {
+int decompress_blob_lz4(const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+#ifdef HAVE_LZ4
+ char* out;
+ uint64_t size;
+ int r;
+
+ assert(src);
+ assert(src_size > 0);
+ assert(dst);
+ assert(dst_alloc_size);
+ assert(dst_size);
+ assert(*dst_alloc_size == 0 || *dst);
+
+ if (src_size <= 8)
+ return -EBADMSG;
+
+ size = le64toh( *(le64_t*)src );
+ if (size > *dst_alloc_size) {
+ out = realloc(*dst, size);
+ if (!out)
+ return -ENOMEM;
+ *dst = out;
+ *dst_alloc_size = size;
+ } else
+ out = *dst;
+
+ r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
+ if (r < 0 || (uint64_t) r != size)
+ return -EBADMSG;
+
+ *dst_size = size;
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_blob(int compression,
+ const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
+ if (compression == OBJECT_COMPRESSED_XZ)
+ return decompress_blob_xz(src, src_size,
+ dst, dst_alloc_size, dst_size, dst_max);
+ else if (compression == OBJECT_COMPRESSED_LZ4)
+ return decompress_blob_lz4(src, src_size,
+ dst, dst_alloc_size, dst_size, dst_max);
+ else
+ return -EBADMSG;
+}
+
+
+int decompress_startswith_xz(const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra) {
+
+#ifdef HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
- /* Checks whether the uncompressed blob starts with the
+ /* Checks whether the decompressed blob starts with the
* mentioned prefix. The byte extra needs to follow the
* prefix */
@@ -133,7 +234,7 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
if (ret != LZMA_OK)
- return false;
+ return -EBADMSG;
if (!(greedy_realloc(buffer, buffer_size, prefix_len + 1, 1)))
return false;
@@ -148,25 +249,61 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
ret = lzma_code(&s, LZMA_FINISH);
if (ret != LZMA_STREAM_END && ret != LZMA_OK)
- return false;
+ return -EBADMSG;
if (*buffer_size - s.avail_out >= prefix_len + 1)
return memcmp(*buffer, prefix, prefix_len) == 0 &&
((const uint8_t*) *buffer)[prefix_len] == extra;
if (ret == LZMA_STREAM_END)
- return false;
+ return -EBADMSG;
+
s.avail_out += *buffer_size;
if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
- return false;
+ return -ENOMEM;
s.next_out = *buffer + *buffer_size - s.avail_out;
}
+
+ return 1;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_startswith_lz4(const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra) {
+#if HAVE_LZ4
+ return -EPROTONOSUPPORT;
+#else
+ return -EPROTONOSUPPORT;
+#endif
}
-int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
+int decompress_startswith(int compression,
+ const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra) {
+ if (compression == OBJECT_COMPRESSED_XZ)
+ return decompress_startswith_xz(src, src_size,
+ buffer, buffer_size,
+ prefix, prefix_len,
+ extra);
+ else if (compression == OBJECT_COMPRESSED_LZ4)
+ return decompress_startswith_lz4(src, src_size,
+ buffer, buffer_size,
+ prefix, prefix_len,
+ extra);
+ else
+ return -EBADMSG;
+}
+
+int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
@@ -176,7 +313,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
- ret = lzma_easy_encoder(&s, preset, LZMA_CHECK_CRC64);
+ ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) {
log_error("Failed to initialize XZ encoder: code %d", ret);
return -EINVAL;
@@ -230,7 +367,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
return errno ? -errno : -EIO;
if (ret == LZMA_STREAM_END) {
- log_debug("Compression finished (%zu -> %zu bytes, %.1f%%)",
+ log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
@@ -240,7 +377,91 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
}
}
-int decompress_stream(int fdf, int fdt, off_t max_bytes) {
+#define LZ4_BUFSIZE (512*1024)
+
+int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_LZ4
+
+ _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
+ char *buf;
+ LZ4_stream_t lz4_data = {};
+ le32_t header;
+ size_t total_in = 0, total_out = sizeof(header);
+ ssize_t n;
+
+ assert(fdf >= 0);
+ assert(fdt >= 0);
+
+ buf1 = malloc(LZ4_BUFSIZE);
+ buf2 = malloc(LZ4_BUFSIZE);
+ out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
+ if (!buf1 || !buf2 || !out)
+ return log_oom();
+
+ buf = buf1;
+ for (;;) {
+ size_t m;
+ int r;
+
+ m = LZ4_BUFSIZE;
+ if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
+ m = max_bytes - total_in;
+
+ n = read(fdf, buf, m);
+ if (n < 0)
+ return -errno;
+ if (n == 0)
+ break;
+
+ total_in += n;
+
+ r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
+ if (r == 0) {
+ log_debug("Compressed size exceeds original, aborting compression.");
+ return -ENOSPC;
+ }
+
+ header = htole32(r);
+ errno = 0;
+
+ n = write(fdt, &header, sizeof(header));
+ if (n < 0)
+ return -errno;
+ if (n != sizeof(header))
+ return errno ? -errno : -EIO;
+
+ n = loop_write(fdt, out, r, false);
+ if (n < 0)
+ return n;
+ if (n != r)
+ return errno ? -errno : -EIO;
+
+ total_out += sizeof(header) + r;
+
+ buf = buf == buf1 ? buf2 : buf1;
+ }
+
+ header = htole32(0);
+ n = write(fdt, &header, sizeof(header));
+ if (n < 0)
+ return -errno;
+ if (n != sizeof(header))
+ return errno ? -errno : -EIO;
+
+ log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
+ total_in, total_out,
+ (double) total_out / total_in * 100);
+
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_XZ
_cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
lzma_ret ret;
@@ -302,7 +523,7 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
return errno ? -errno : -EIO;
if (ret == LZMA_STREAM_END) {
- log_debug("Decompression finished (%zu -> %zu bytes, %.1f%%)",
+ log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
@@ -310,4 +531,99 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
}
}
}
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
+
+#ifdef HAVE_LZ4
+ _cleanup_free_ char *buf = NULL, *out = NULL;
+ size_t buf_size = 0;
+ LZ4_streamDecode_t lz4_data = {};
+ le32_t header;
+ size_t total_in = sizeof(header), total_out = 0;
+
+ assert(fdf >= 0);
+ assert(fdt >= 0);
+
+ out = malloc(4*LZ4_BUFSIZE);
+ if (!out)
+ return log_oom();
+
+ for (;;) {
+ ssize_t n, m;
+ int r;
+
+ n = read(fdf, &header, sizeof(header));
+ if (n < 0)
+ return -errno;
+ if (n != sizeof(header))
+ return errno ? -errno : -EIO;
+
+ m = le32toh(header);
+ if (m == 0)
+ break;
+
+ /* We refuse to use a bigger decompression buffer than
+ * the one used for compression by 4 times. This means
+ * that compression buffer size can be enlarged 4
+ * times. This can be changed, but old binaries might
+ * not accept buffers compressed by newer binaries then.
+ */
+ if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
+ log_error("Compressed stream block too big: %zd bytes", m);
+ return -EBADMSG;
+ }
+
+ total_in += sizeof(header) + m;
+
+ if (!GREEDY_REALLOC(buf, buf_size, m))
+ return log_oom();
+
+ errno = 0;
+ n = loop_read(fdf, buf, m, false);
+ if (n < 0)
+ return n;
+ if (n != m)
+ return errno ? -errno : -EIO;
+
+ r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
+ if (r <= 0)
+ log_error("LZ4 decompression failed.");
+
+ total_out += r;
+
+ if (max_bytes != -1 && total_out > (size_t) max_bytes) {
+ log_debug("Decompressed stream longer than %zd bytes", max_bytes);
+ return -E2BIG;
+ }
+
+ errno = 0;
+ n = loop_write(fdt, out, r, false);
+ if (n < 0)
+ return n;
+ if (n != r)
+ return errno ? -errno : -EIO;
+ }
+
+ log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
+ total_in, total_out,
+ (double) total_out / total_in * 100);
+
+ return 0;
+#else
+ return -EPROTONOSUPPORT;
+#endif
+}
+
+int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
+
+ if (endswith(filename, ".lz4"))
+ return decompress_stream_lz4(fdf, fdt, max_bytes);
+ else if (endswith(filename, ".xz"))
+ return decompress_stream_xz(fdf, fdt, max_bytes);
+ else
+ return -EPROTONOSUPPORT;
}
diff --git a/src/journal/compress.h b/src/journal/compress.h
index f25fe86abd..0810a13c3f 100644
--- a/src/journal/compress.h
+++ b/src/journal/compress.h
@@ -25,15 +25,62 @@
#include <stdbool.h>
#include <unistd.h>
-bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
+#include "journal-def.h"
-bool uncompress_blob(const void *src, uint64_t src_size,
- void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+const char* object_compressed_to_string(int compression);
+int object_compressed_from_string(const char *compression);
-bool uncompress_startswith(const void *src, uint64_t src_size,
- void **buffer, uint64_t *buffer_size,
- const void *prefix, uint64_t prefix_len,
- uint8_t extra);
+int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
+int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
-int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_size);
-int decompress_stream(int fdf, int fdt, off_t max_size);
+static inline int compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
+ int r;
+#if HAVE_LZ4
+ r = compress_blob_lz4(src, src_size, dst, dst_size);
+ if (r == 0)
+ return OBJECT_COMPRESSED_LZ4;
+#else
+ r = compress_blob_xz(src, src_size, dst, dst_size);
+ if (r == 0)
+ return OBJECT_COMPRESSED_XZ;
+#endif
+ return r;
+}
+
+int decompress_blob_xz(const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+int decompress_blob_lz4(const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+int decompress_blob(int compression,
+ const void *src, uint64_t src_size,
+ void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
+
+int decompress_startswith_xz(const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra);
+int decompress_startswith_lz4(const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra);
+int decompress_startswith(int compression,
+ const void *src, uint64_t src_size,
+ void **buffer, uint64_t *buffer_size,
+ const void *prefix, uint64_t prefix_len,
+ uint8_t extra);
+
+int compress_stream_xz(int fdf, int fdt, off_t max_bytes);
+int compress_stream_lz4(int fdf, int fdt, off_t max_bytes);
+
+int decompress_stream_xz(int fdf, int fdt, off_t max_size);
+int decompress_stream_lz4(int fdf, int fdt, off_t max_size);
+
+#if HAVE_LZ4
+# define compress_stream compress_stream_lz4
+# define COMPRESSED_EXT ".lz4"
+#else
+# define compress_stream compress_stream_lz4
+# define COMPRESSED_EXT ".xz"
+#endif
+
+int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes);
diff --git a/src/journal/coredump.c b/src/journal/coredump.c
index 78e89d33cf..e46719706d 100644
--- a/src/journal/coredump.c
+++ b/src/journal/coredump.c
@@ -357,7 +357,7 @@ static int save_external_coredump(
goto fail;
}
-#ifdef HAVE_XZ
+#if defined(HAVE_XZ) || defined(HAVE_LZ4)
/* If we will remove the coredump anyway, do not compress. */
if (maybe_remove_external_coredump(NULL, st.st_size) == 0
&& arg_compress) {
@@ -365,15 +365,15 @@ static int save_external_coredump(
_cleanup_free_ char *fn_compressed = NULL, *tmp_compressed = NULL;
_cleanup_close_ int fd_compressed = -1;
- fn_compressed = strappend(fn, ".xz");
+ fn_compressed = strappend(fn, COMPRESSED_EXT);
if (!fn_compressed) {
- r = log_oom();
+ log_oom();
goto uncompressed;
}
tmp_compressed = tempfn_random(fn_compressed);
if (!tmp_compressed) {
- r = log_oom();
+ log_oom();
goto uncompressed;
}
@@ -383,7 +383,7 @@ static int save_external_coredump(
goto uncompressed;
}
- r = compress_stream(fd, fd_compressed, LZMA_PRESET_DEFAULT, -1);
+ r = compress_stream(fd, fd_compressed, -1);
if (r < 0) {
log_error("Failed to compress %s: %s", tmp_compressed, strerror(-r));
goto fail_compressed;
diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c
index 2158d73771..39f6e65575 100644
--- a/src/journal/coredumpctl.c
+++ b/src/journal/coredumpctl.c
@@ -656,7 +656,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) {
goto error;
}
- r = decompress_stream(fdf, fd, -1);
+ r = decompress_stream(filename, fdf, fd, -1);
if (r < 0) {
log_error("Failed to decompress %s: %s", filename, strerror(-r));
goto error;
diff --git a/src/journal/journal-def.h b/src/journal/journal-def.h
index 7e407a416c..ecfa9a2b16 100644
--- a/src/journal/journal-def.h
+++ b/src/journal/journal-def.h
@@ -66,9 +66,13 @@ enum {
/* Object flags */
enum {
- OBJECT_COMPRESSED = 1
+ OBJECT_COMPRESSED_XZ = 1 << 0,
+ OBJECT_COMPRESSED_LZ4 = 1 << 1,
+ _OBJECT_COMPRESSED_MAX
};
+#define OBJECT_COMPRESSION_MASK (OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4)
+
struct ObjectHeader {
uint8_t type;
uint8_t flags;
@@ -155,13 +159,33 @@ enum {
/* Header flags */
enum {
- HEADER_INCOMPATIBLE_COMPRESSED = 1
+ HEADER_INCOMPATIBLE_COMPRESSED_XZ = 1 << 0,
+ HEADER_INCOMPATIBLE_COMPRESSED_LZ4 = 1 << 1,
};
+#define HEADER_INCOMPATIBLE_ANY (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
+
+#if defined(HAVE_XZ) && defined(HAVE_LZ4)
+# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_ANY
+#elif defined(HAVE_XZ)
+# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_XZ
+#elif defined(HAVE_LZ4)
+# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_LZ4
+#else
+# define HEADER_INCOMPATIBLE_SUPPORTED 0
+#endif
+
enum {
HEADER_COMPATIBLE_SEALED = 1
};
+#define HEADER_COMPATIBLE_ANY HEADER_COMPATIBLE_SEALED
+#if HAVE_GCRYPT
+# define HEADER_COMPATIBLE_SUPPORTED HEADER_COMPATIBLE_SEALED
+#else
+# define HEADER_COMPATIBLE_SUPPORTED 0
+#endif
+
#define HEADER_SIGNATURE ((char[]) { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' })
struct Header {
diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index b3b1ffc3c0..b8d375e1d2 100644
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -158,21 +158,21 @@ void journal_file_close(JournalFile *f) {
}
static int journal_file_init_header(JournalFile *f, JournalFile *template) {
- Header h;
+ Header h = {};
ssize_t k;
int r;
assert(f);
- zero(h);
memcpy(h.signature, HEADER_SIGNATURE, 8);
h.header_size = htole64(ALIGN64(sizeof(h)));
- h.incompatible_flags =
- htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
+ h.incompatible_flags |= htole32(
+ f->compress_xz * HEADER_INCOMPATIBLE_COMPRESSED_XZ |
+ f->compress_lz4 * HEADER_INCOMPATIBLE_COMPRESSED_LZ4);
- h.compatible_flags =
- htole32(f->seal ? HEADER_COMPATIBLE_SEALED : 0);
+ h.compatible_flags = htole32(
+ f->seal * HEADER_COMPATIBLE_SEALED);
r = sd_id128_randomize(&h.file_id);
if (r < 0)
@@ -222,6 +222,8 @@ static int journal_file_refresh_header(JournalFile *f) {
}
static int journal_file_verify_header(JournalFile *f) {
+ uint32_t flags;
+
assert(f);
if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
@@ -229,24 +231,30 @@ static int journal_file_verify_header(JournalFile *f) {
/* In both read and write mode we refuse to open files with
* incompatible flags we don't know */
-#ifdef HAVE_XZ
- if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
+ flags = le32toh(f->header->incompatible_flags);
+ if (flags & ~HEADER_INCOMPATIBLE_SUPPORTED) {
+ if (flags & ~HEADER_INCOMPATIBLE_ANY)
+ log_debug("Journal file %s has unknown incompatible flags %"PRIx32,
+ f->path, flags & ~HEADER_INCOMPATIBLE_ANY);
+ flags = (flags & HEADER_INCOMPATIBLE_ANY) & ~HEADER_INCOMPATIBLE_SUPPORTED;
+ if (flags)
+ log_debug("Journal file %s uses incompatible flags %"PRIx32
+ " disabled at compilation time.", f->path, flags);
return -EPROTONOSUPPORT;
-#else
- if (f->header->incompatible_flags != 0)
- return -EPROTONOSUPPORT;
-#endif
+ }
/* When open for writing we refuse to open files with
* compatible flags, too */
- if (f->writable) {
-#ifdef HAVE_GCRYPT
- if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) != 0)
- return -EPROTONOSUPPORT;
-#else
- if (f->header->compatible_flags != 0)
- return -EPROTONOSUPPORT;
-#endif
+ flags = le32toh(f->header->compatible_flags);
+ if (f->writable && (flags & ~HEADER_COMPATIBLE_SUPPORTED)) {
+ if (flags & ~HEADER_COMPATIBLE_ANY)
+ log_debug("Journal file %s has unknown compatible flags %"PRIx32,
+ f->path, flags & ~HEADER_COMPATIBLE_ANY);
+ flags = (flags & HEADER_COMPATIBLE_ANY) & ~HEADER_COMPATIBLE_SUPPORTED;
+ if (flags)
+ log_debug("Journal file %s uses compatible flags %"PRIx32
+ " disabled at compilation time.", f->path, flags);
+ return -EPROTONOSUPPORT;
}
if (f->header->state >= _STATE_MAX)
@@ -302,7 +310,8 @@ static int journal_file_verify_header(JournalFile *f) {
}
}
- f->compress = JOURNAL_HEADER_COMPRESSED(f->header);
+ f->compress_xz = JOURNAL_HEADER_COMPRESSED_XZ(f->header);
+ f->compress_lz4 = JOURNAL_HEADER_COMPRESSED_LZ4(f->header);
f->seal = JOURNAL_HEADER_SEALED(f->header);
@@ -809,8 +818,7 @@ int journal_file_find_data_object_with_hash(
if (le64toh(o->data.hash) != hash)
goto next;
- if (o->object.flags & OBJECT_COMPRESSED) {
-#ifdef HAVE_XZ
+ if (o->object.flags & OBJECT_COMPRESSION_MASK) {
uint64_t l, rsize;
l = le64toh(o->object.size);
@@ -819,8 +827,10 @@ int journal_file_find_data_object_with_hash(
l -= offsetof(Object, data.payload);
- if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0))
- return -EBADMSG;
+ r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
+ o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0);
+ if (r < 0)
+ return r;
if (rsize == size &&
memcmp(f->compress_buffer, data, size) == 0) {
@@ -833,9 +843,6 @@ int journal_file_find_data_object_with_hash(
return 1;
}
-#else
- return -EPROTONOSUPPORT;
-#endif
} else if (le64toh(o->object.size) == osize &&
memcmp(o->data.payload, data, size) == 0) {
@@ -943,8 +950,7 @@ static int journal_file_append_data(
uint64_t hash, p;
uint64_t osize;
Object *o;
- int r;
- bool compressed = false;
+ int r, compression = 0;
const void *eq;
assert(f);
@@ -974,22 +980,23 @@ static int journal_file_append_data(
o->data.hash = htole64(hash);
#ifdef HAVE_XZ
- if (f->compress &&
+ if (f->compress_xz &&
size >= COMPRESSION_SIZE_THRESHOLD) {
uint64_t rsize;
- compressed = compress_blob(data, size, o->data.payload, &rsize);
+ compression = compress_blob(data, size, o->data.payload, &rsize);
- if (compressed) {
+ if (compression) {
o->object.size = htole64(offsetof(Object, data.payload) + rsize);
- o->object.flags |= OBJECT_COMPRESSED;
+ o->object.flags |= compression;
- log_debug("Compressed data object %"PRIu64" -> %"PRIu64, size, rsize);
+ log_debug("Compressed data object %"PRIu64" -> %"PRIu64" using %s",
+ size, rsize, object_compressed_to_string(compression));
}
}
#endif
- if (!compressed && size > 0)
+ if (!compression && size > 0)
memcpy(o->data.payload, data, size);
r = journal_file_link_data(f, o, p, hash);
@@ -2332,8 +2339,9 @@ void journal_file_dump(JournalFile *f) {
break;
}
- if (o->object.flags & OBJECT_COMPRESSED)
- printf("Flags: COMPRESSED\n");
+ if (o->object.flags & OBJECT_COMPRESSION_MASK)
+ printf("Flags: %s\n",
+ object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK));
if (p == le64toh(f->header->tail_object_offset))
p = 0;
@@ -2370,7 +2378,7 @@ void journal_file_print_header(JournalFile *f) {
"Sequential Number ID: %s\n"
"State: %s\n"
"Compatible Flags:%s%s\n"
- "Incompatible Flags:%s%s\n"
+ "Incompatible Flags:%s%s%s\n"
"Header size: %"PRIu64"\n"
"Arena size: %"PRIu64"\n"
"Data Hash Table Size: %"PRIu64"\n"
@@ -2392,9 +2400,10 @@ void journal_file_print_header(JournalFile *f) {
f->header->state == STATE_ONLINE ? "ONLINE" :
f->header->state == STATE_ARCHIVED ? "ARCHIVED" : "UNKNOWN",
JOURNAL_HEADER_SEALED(f->header) ? " SEALED" : "",
- (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) ? " ???" : "",
- JOURNAL_HEADER_COMPRESSED(f->header) ? " COMPRESSED" : "",
- (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "",
+ (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_ANY) ? " ???" : "",
+ JOURNAL_HEADER_COMPRESSED_XZ(f->header) ? " COMPRESSED-XZ" : "",
+ JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "",
+ (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_ANY) ? " ???" : "",
le64toh(f->header->header_size),
le64toh(f->header->arena_size),
le64toh(f->header->data_hash_table_size) / sizeof(HashItem),
@@ -2468,7 +2477,7 @@ int journal_file_open(
f->prot = prot_from_flags(flags);
f->writable = (flags & O_ACCMODE) != O_RDONLY;
#ifdef HAVE_XZ
- f->compress = compress;
+ f->compress_xz = compress;
#endif
#ifdef HAVE_GCRYPT
f->seal = seal;
@@ -2760,11 +2769,11 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
if ((uint64_t) t != l)
return -E2BIG;
- if (o->object.flags & OBJECT_COMPRESSED) {
+ if (o->object.flags & OBJECT_COMPRESSED_XZ) {
#ifdef HAVE_XZ
uint64_t rsize;
- if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
+ if (!decompress_blob_xz(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
return -EBADMSG;
data = from->compress_buffer;
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index 50dbe29cc1..b0c28b5e83 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -56,7 +56,8 @@ typedef struct JournalFile {
int flags;
int prot;
bool writable:1;
- bool compress:1;
+ bool compress_xz:1;
+ bool compress_lz4:1;
bool seal:1;
bool tail_entry_monotonic_valid:1;
@@ -153,8 +154,11 @@ static inline bool VALID_EPOCH(uint64_t u) {
#define JOURNAL_HEADER_SEALED(h) \
(!!(le32toh((h)->compatible_flags) & HEADER_COMPATIBLE_SEALED))
-#define JOURNAL_HEADER_COMPRESSED(h) \
- (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED))
+#define JOURNAL_HEADER_COMPRESSED_XZ(h) \
+ (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_XZ))
+
+#define JOURNAL_HEADER_COMPRESSED_LZ4(h) \
+ (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_LZ4))
int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c
index 31bae5a8f8..c063d12210 100644
--- a/src/journal/journal-verify.c
+++ b/src/journal/journal-verify.c
@@ -45,7 +45,7 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
* possible field values. It does not follow any references to
* other objects. */
- if ((o->object.flags & OBJECT_COMPRESSED) &&
+ if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
o->object.type != OBJECT_DATA)
return -EBADMSG;
@@ -72,22 +72,38 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
h1 = le64toh(o->data.hash);
- if (o->object.flags & OBJECT_COMPRESSED) {
+ if (o->object.flags & OBJECT_COMPRESSED_XZ) {
#ifdef HAVE_XZ
- void *b = NULL;
+ _cleanup_free_ void *b = NULL;
uint64_t alloc = 0, b_size;
- if (!uncompress_blob(o->data.payload,
- le64toh(o->object.size) - offsetof(Object, data.payload),
- &b, &alloc, &b_size, 0)) {
- log_error(OFSfmt": uncompression failed", offset);
+ if (!decompress_blob_xz(o->data.payload,
+ le64toh(o->object.size) - offsetof(Object, data.payload),
+ &b, &alloc, &b_size, 0)) {
+ log_error(OFSfmt": XZ decompression failed", offset);
return -EBADMSG;
}
h2 = hash64(b, b_size);
- free(b);
#else
- log_error("Compression is not supported");
+ log_error("XZ compression is not supported");
+ return -EPROTONOSUPPORT;
+#endif
+ } else if (o->object.flags & OBJECT_COMPRESSED_LZ4) {
+#ifdef HAVE_XZ
+ _cleanup_free_ void *b = NULL;
+ uint64_t alloc = 0, b_size;
+
+ if (!decompress_blob_xz(o->data.payload,
+ le64toh(o->object.size) - offsetof(Object, data.payload),
+ &b, &alloc, &b_size, 0)) {
+ log_error(OFSfmt": LZ4 decompression failed", offset);
+ return -EBADMSG;
+ }
+
+ h2 = hash64(b, b_size);
+#else
+ log_error("XZ compression is not supported");
return -EPROTONOSUPPORT;
#endif
} else
@@ -875,8 +891,21 @@ int journal_file_verify(
goto fail;
}
- if ((o->object.flags & OBJECT_COMPRESSED) && !JOURNAL_HEADER_COMPRESSED(f->header)) {
- log_error("Compressed object in file without compression at "OFSfmt, p);
+ if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
+ (o->object.flags & OBJECT_COMPRESSED_LZ4)) {
+ log_error("Objected with double compression at "OFSfmt, p);
+ r = -EBADMSG;
+ goto fail;
+ }
+
+ if ((o->object.flags & OBJECT_COMPRESSED_XZ) && !JOURNAL_HEADER_COMPRESSED_XZ(f->header)) {
+ log_error("XZ compressed object in file without XZ compression at "OFSfmt, p);
+ r = -EBADMSG;
+ goto fail;
+ }
+
+ if ((o->object.flags & OBJECT_COMPRESSED_LZ4) && !JOURNAL_HEADER_COMPRESSED_LZ4(f->header)) {
+ log_error("LZ4 compressed object in file without LZ4 compression at "OFSfmt, p);
r = -EBADMSG;
goto fail;
}
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index ca805f83fe..96ba2bd11e 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1995,28 +1995,24 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
l = le64toh(o->object.size) - offsetof(Object, data.payload);
- if (o->object.flags & OBJECT_COMPRESSED) {
+ if ((o->object.flags & OBJECT_COMPRESSION_MASK) &&
+ decompress_startswith(o->object.flags & OBJECT_COMPRESSION_MASK,
+ o->data.payload, l,
+ &f->compress_buffer, &f->compress_buffer_size,
+ field, field_length, '=')) {
-#ifdef HAVE_XZ
- if (uncompress_startswith(o->data.payload, l,
- &f->compress_buffer, &f->compress_buffer_size,
- field, field_length, '=')) {
-
- uint64_t rsize;
+ uint64_t rsize;
- if (!uncompress_blob(o->data.payload, l,
- &f->compress_buffer, &f->compress_buffer_size, &rsize,
- j->data_threshold))
- return -EBADMSG;
+ r = decompress_blob_xz(o->data.payload, l,
+ &f->compress_buffer, &f->compress_buffer_size, &rsize,
+ j->data_threshold);
+ if (r < 0)
+ return r;
- *data = f->compress_buffer;
- *size = (size_t) rsize;
+ *data = f->compress_buffer;
+ *size = (size_t) rsize;
- return 0;
- }
-#else
- return -EPROTONOSUPPORT;
-#endif
+ return 0;
} else if (l >= field_length+1 &&
memcmp(o->data.payload, field, field_length) == 0 &&
@@ -2052,11 +2048,11 @@ static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **da
if ((uint64_t) t != l)
return -E2BIG;
- if (o->object.flags & OBJECT_COMPRESSED) {
+ if (o->object.flags & OBJECT_COMPRESSED_XZ) {
#ifdef HAVE_XZ
uint64_t rsize;
- if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
+ if (!decompress_blob_xz(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
return -EBADMSG;
*data = f->compress_buffer;
diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c
index 0806145d4b..24171463a2 100644
--- a/src/journal/test-compress.c
+++ b/src/journal/test-compress.c
@@ -21,54 +21,110 @@
#include "util.h"
#include "macro.h"
-static void test_compress_uncompress(void) {
+#ifdef HAVE_XZ
+# define XZ_OK 0
+#else
+# define XZ_OK -EPROTONOSUPPORT
+#endif
+
+#ifdef HAVE_XZ
+# define LZ4_OK 0
+#else
+# define LZ4_OK -EPROTONOSUPPORT
+#endif
+
+static void test_compress_decompress(void) {
char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
"foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
char compressed[512];
uint64_t csize = 512;
uint64_t usize = 0;
- _cleanup_free_ char *uncompressed = NULL;
-
- assert_se(compress_blob(text, sizeof(text), compressed, &csize));
- assert_se(uncompress_blob(compressed,
- csize,
- (void **) &uncompressed,
- &usize, &csize, 0));
- assert_se(uncompressed);
- assert_se(streq(uncompressed, text));
- assert_se(!uncompress_blob("garbage",
- 7,
- (void **) &uncompressed,
- &usize, &csize, 0));
+ _cleanup_free_ char *decompressed = NULL, *decompressed2 = NULL;
+ int r;
+
+ r = compress_blob_xz(text, sizeof(text), compressed, &csize);
+ assert(r == XZ_OK);
+ r = decompress_blob_xz(compressed,
+ csize,
+ (void **) &decompressed,
+ &usize, &csize, 0);
+ assert(r == XZ_OK);
+#ifdef HAVE_XZ
+ assert_se(decompressed);
+ assert_se(streq(decompressed, text));
+#endif
+ r = decompress_blob_xz("garbage",
+ 7,
+ (void **) &decompressed,
+ &usize, &csize, 0);
+ assert(r < 0);
+
+
+ r = compress_blob_lz4(text, sizeof(text), compressed, &csize);
+ assert(r == LZ4_OK);
+ r = decompress_blob_lz4(compressed,
+ csize,
+ (void **) &decompressed2,
+ &usize, &csize, 0);
+ assert(r == LZ4_OK);
+#ifdef HAVE_LZ4
+ assert_se(decompressed2);
+ assert_se(streq(decompressed2, text));
+#endif
+
+ r = decompress_blob_lz4("garbage",
+ 7,
+ (void **) &decompressed2,
+ &usize, &csize, 0);
+ assert(r < 0);
+
+ r = decompress_blob_lz4("00000000\1g",
+ 9,
+ (void **) &decompressed2,
+ &usize, &csize, 0);
+ assert(r < 0);
}
-static void test_uncompress_startswith(void) {
+static void test_decompress_startswith(void) {
char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
"foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
char compressed[512];
uint64_t csize = 512;
uint64_t usize = 0;
- _cleanup_free_ char *uncompressed = NULL;
-
- assert_se(compress_blob(text, sizeof(text), compressed, &csize));
- assert_se(uncompress_startswith(compressed,
- csize,
- (void **) &uncompressed,
- &usize,
- "foofoofoofoo", 12, ' '));
- assert_se(!uncompress_startswith(compressed,
- csize,
- (void **) &uncompressed,
- &usize,
- "foofoofoofoo", 12, 'w'));
- assert_se(!uncompress_startswith(compressed,
- csize,
- (void **) &uncompressed,
- &usize,
- "barbarbar", 9, ' '));
+ _cleanup_free_ char *decompressed = NULL;
+
+ assert_se(compress_blob_xz(text, sizeof(text), compressed, &csize) == 0);
+ assert_se(decompress_startswith_xz(compressed,
+ csize,
+ (void **) &decompressed,
+ &usize,
+ "foofoofoofoo", 12, ' ') > 0);
+ assert_se(decompress_startswith_xz(compressed,
+ csize,
+ (void **) &decompressed,
+ &usize,
+ "foofoofoofoo", 12, 'w') == 0);
+ assert_se(decompress_startswith_xz(compressed,
+ csize,
+ (void **) &decompressed,
+ &usize,
+ "barbarbar", 9, ' ') == 0);
+ assert_se(decompress_startswith_xz(compressed,
+ csize,
+ (void **) &decompressed,
+ &usize,
+ "foofoofoofoo", 12, ' ') > 0);
}
-static void test_compress_stream(const char *srcfile) {
+typedef int (compress_stream_t)(int fdf, int fdt, off_t max_bytes);
+typedef int (decompress_stream_t)(int fdf, int fdt, off_t max_size);
+
+static void test_compress_stream(const char* type,
+ const char* cat,
+ compress_stream_t compress,
+ decompress_stream_t decompress,
+ const char *srcfile) {
+
_cleanup_close_ int src = -1, dst = -1, dst2 = -1;
char pattern[] = "/tmp/systemd-test.xz.XXXXXX",
pattern2[] = "/tmp/systemd-test.xz.XXXXXX";
@@ -76,6 +132,8 @@ static void test_compress_stream(const char *srcfile) {
_cleanup_free_ char *cmd, *cmd2;
struct stat st = {};
+ log_debug("/* testing %s compression */", type);
+
log_debug("/* create source from %s */", srcfile);
assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
@@ -84,11 +142,13 @@ static void test_compress_stream(const char *srcfile) {
assert_se((dst = mkostemp_safe(pattern, O_RDWR|O_CLOEXEC)) >= 0);
- r = compress_stream(src, dst, 1, -1);
+ r = compress(src, dst, -1);
+ log_debug("compress -> %s", strerror(-r));
assert(r == 0);
- assert_se(asprintf(&cmd, "xzcat %s | diff %s -", pattern, srcfile) > 0);
- assert_se(system(cmd) == 0);
+ assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
+ if (system(cmd) == 0)
+ log_warning("External decompression failed.");
log_debug("/* test decompression */");
@@ -97,7 +157,7 @@ static void test_compress_stream(const char *srcfile) {
assert_se(stat(srcfile, &st) == 0);
assert_se(lseek(dst, 0, SEEK_SET) == 0);
- r = decompress_stream(dst, dst2, st.st_size);
+ r = decompress(dst, dst2, st.st_size);
assert(r == 0);
assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
@@ -106,12 +166,12 @@ static void test_compress_stream(const char *srcfile) {
log_debug("/* test faulty decompression */");
assert_se(lseek(dst, 1, SEEK_SET) == 1);
- r = decompress_stream(dst, dst2, st.st_size);
+ r = decompress(dst, dst2, st.st_size);
assert(r == -EBADMSG);
assert_se(lseek(dst, 0, SEEK_SET) == 0);
assert_se(lseek(dst2, 0, SEEK_SET) == 0);
- r = decompress_stream(dst, dst2, st.st_size - 1);
+ r = decompress(dst, dst2, st.st_size - 1);
assert(r == -E2BIG);
assert_se(unlink(pattern) == 0);
@@ -122,9 +182,21 @@ int main(int argc, char *argv[]) {
log_set_max_level(LOG_DEBUG);
- test_compress_uncompress();
- test_uncompress_startswith();
- test_compress_stream(argv[0]);
+ test_compress_decompress();
+ test_decompress_startswith();
+
+#ifdef HAVE_XZ
+ test_compress_stream("XZ", "xzcat",
+ compress_stream_xz, decompress_stream_xz, argv[0]);
+#else
+ log_info("/* XZ test skipped */");
+#endif
+#ifdef HAVE_LZ4
+ test_compress_stream("LZ4", "lz4cat",
+ compress_stream_lz4, decompress_stream_lz4, argv[0]);
+#else
+ log_info("/* LZ4 test skipped */");
+#endif
return 0;
}
--
2.0.0
More information about the systemd-devel
mailing list