[systemd-devel] [PATCH 1/2] journal: add LZ4 as optional compressor
Ronny Chevalier
chevalier.ronny at gmail.com
Sun Jul 6 03:43:06 PDT 2014
2014-07-05 20:56 GMT+02:00 Zbigniew Jędrzejewski-Szmek <zbyszek at in.waw.pl>:
> Add liblz4 as an optional dependency when requested with --enable-lz4, and
> use it in preference to liblzma for journal blob and coredump compression. To
> retain backwards compatibility, XZ is used to decompress old blobs.
>
> Things will function correctly only with lz4-119.
>
> Based on the benchmarks found on the web, lz4 seems to be the best choice for
> "quick" compressors atm. If we decide to go this way, I'll work with upstream
> to improve their packaging practices, and at least use pkg-config.
>
> ---
>
> Makefile.am | 11 +-
> configure.ac | 15 +-
> src/journal/compress.c | 382 +++++++++++++++++++++++++++++++++++++++----
> src/journal/compress.h | 65 +++++++-
> src/journal/coredump.c | 10 +-
> src/journal/coredumpctl.c | 2 +-
> src/journal/journal-def.h | 28 +++-
> src/journal/journal-file.c | 99 ++++++-----
> src/journal/journal-file.h | 10 +-
> src/journal/journal-verify.c | 51 ++++--
> src/journal/sd-journal.c | 36 ++--
> src/journal/test-compress.c | 158 +++++++++++++-----
> 12 files changed, 688 insertions(+), 179 deletions(-)
>
> diff --git a/Makefile.am b/Makefile.am
> index 55a7546eef..c85d66ef32 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -3526,14 +3526,12 @@ test_catalog_CPPFLAGS = \
> test_catalog_LDADD = \
> libsystemd-journal-core.la
>
> -if HAVE_XZ
> test_compress_SOURCES = \
> src/journal/test-compress.c
>
> test_compress_LDADD = \
> libsystemd-journal-internal.la \
> libsystemd-shared.la
> -endif
>
> libsystemd_journal_core_la_SOURCES = \
> src/journal/journald-kmsg.c \
> @@ -3617,9 +3615,7 @@ tests += \
> test-mmap-cache \
> test-catalog
>
> -if HAVE_XZ
> tests += test-compress
> -endif
>
> pkginclude_HEADERS += \
> src/systemd/sd-journal.h \
> @@ -3652,10 +3648,10 @@ libsystemd_journal_internal_la_CFLAGS = \
>
> libsystemd_journal_internal_la_LIBADD =
>
> -if HAVE_XZ
> libsystemd_journal_internal_la_SOURCES += \
> src/journal/compress.c
>
> +if HAVE_XZ
> libsystemd_journal_internal_la_CFLAGS += \
> $(XZ_CFLAGS)
>
> @@ -3663,6 +3659,11 @@ libsystemd_journal_internal_la_LIBADD += \
> $(XZ_LIBS)
> endif
>
> +if HAVE_LZ4
> +libsystemd_journal_internal_la_LIBADD += \
> + -llz4
> +endif
> +
> if HAVE_GCRYPT
> libsystemd_journal_internal_la_SOURCES += \
> src/journal/journal-authenticate.c \
> diff --git a/configure.ac b/configure.ac
> index 93aba06739..7a04ca4fa0 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -503,14 +503,24 @@ have_xz=no
> AC_ARG_ENABLE(xz, AS_HELP_STRING([--disable-xz], [Disable optional XZ support]))
> if test "x$enable_xz" != "xno"; then
> PKG_CHECK_MODULES(XZ, [ liblzma ],
> - [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes], have_xz=no)
> + [AC_DEFINE(HAVE_XZ, 1, [Define if XZ is available]) have_xz=yes])
> if test "x$have_xz" = xno -a "x$enable_xz" = xyes; then
> - AC_MSG_ERROR([*** Xz support requested but libraries not found])
> + AC_MSG_ERROR([*** XZ support requested but libraries not found])
> fi
> fi
> AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
>
> # ------------------------------------------------------------------------------
> +have_lz4=no
> +AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
> +AS_IF([test "x$enable_lz4" == "xyes"], [
> + AC_CHECK_HEADERS(lz4.h,
> + [AC_DEFINE(HAVE_LZ4, 1, [Define in LZ4 is available]) have_lz4=yes],
> + [AC_MSG_ERROR([*** LZ4 support requested but headers not found])])
> +])
> +AM_CONDITIONAL(HAVE_LZ4, [test "$have_lz4" = "yes"])
> +
> +# ------------------------------------------------------------------------------
> AC_ARG_ENABLE([pam],
> AS_HELP_STRING([--disable-pam],[Disable optional PAM support]),
> [case "${enableval}" in
> @@ -1266,6 +1276,7 @@ AC_MSG_RESULT([
> SECCOMP: ${have_seccomp}
> SMACK: ${have_smack}
> XZ: ${have_xz}
> + LZ4: ${have_lz4}
> ACL: ${have_acl}
> GCRYPT: ${have_gcrypt}
> QRENCODE: ${have_qrencode}
> diff --git a/src/journal/compress.c b/src/journal/compress.c
> index 37c55a8728..f7fc665e9c 100644
> --- a/src/journal/compress.c
> +++ b/src/journal/compress.c
> @@ -23,13 +23,30 @@
> #include <stdlib.h>
> #include <string.h>
> #include <unistd.h>
> -#include <lzma.h>
> +
> +#ifdef HAVE_XZ
> +# include <lzma.h>
> +#endif
> +
> +#ifdef HAVE_LZ4
> +# include <lz4.h>
> +#endif
>
> #include "compress.h"
> #include "macro.h"
> #include "util.h"
> +#include "sparse-endian.h"
> +#include "journal-def.h"
> +
> +static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
> + [OBJECT_COMPRESSED_XZ] = "XZ",
> + [OBJECT_COMPRESSED_LZ4] = "LZ4",
> +};
> +
> +DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
>
> -bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
> +int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
> +#ifdef HAVE_XZ
> lzma_ret ret;
> size_t out_pos = 0;
>
> @@ -42,21 +59,50 @@ bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_
> * compressed result is longer than the original */
>
> ret = lzma_easy_buffer_encode(LZMA_PRESET_DEFAULT, LZMA_CHECK_NONE, NULL,
> - src, src_size, dst, &out_pos, src_size);
> + src, src_size, dst, &out_pos, src_size - 1);
> if (ret != LZMA_OK)
> - return false;
> -
> - /* Is it actually shorter? */
> - if (out_pos == src_size)
> - return false;
> + return -ENOSPC;
>
> *dst_size = out_pos;
> - return true;
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> }
>
> -bool uncompress_blob(const void *src, uint64_t src_size,
> - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
> +int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
> +#ifdef HAVE_LZ4
> + int r;
>
> + assert(src);
> + assert(src_size > 0);
> + assert(dst);
> + assert(dst_size);
> +
> + /* Returns false if we couldn't compress the data or the
> + * compressed result is longer than the original */
Obsolete comment
> +
> + if (src_size < 9)
> + return -ENOSPC;
> +
> + r = LZ4_compress_limitedOutput(src, dst + 8, src_size, src_size - 8 - 1);
> + if (r <= 0)
> + return -ENOSPC;
> +
> + *(le64_t*) dst = htole64(src_size);
> + *dst_size = r + 8;
> +
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +
> +int decompress_blob_xz(const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
> +
> +#ifdef HAVE_XZ
> _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
> lzma_ret ret;
> uint64_t space;
> @@ -70,7 +116,7 @@ bool uncompress_blob(const void *src, uint64_t src_size,
>
> ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
> if (ret != LZMA_OK)
> - return false;
> + return -ENOMEM;
>
> space = MIN(src_size * 2, dst_max ?: (uint64_t) -1);
> if (!greedy_realloc(dst, dst_alloc_size, space, 1))
> @@ -89,15 +135,13 @@ bool uncompress_blob(const void *src, uint64_t src_size,
>
> if (ret == LZMA_STREAM_END)
> break;
> -
> - if (ret != LZMA_OK)
> - return false;
> + else if (ret != LZMA_OK)
> + return -ENOMEM;
>
> if (dst_max > 0 && (space - s.avail_out) >= dst_max)
> break;
> -
> - if (dst_max > 0 && space == dst_max)
> - return false;
> + else if (dst_max > 0 && space == dst_max)
> + return -E2BIG;
>
> used = space - s.avail_out;
> space = MIN(2 * space, dst_max ?: (uint64_t) -1);
> @@ -109,18 +153,75 @@ bool uncompress_blob(const void *src, uint64_t src_size,
> }
>
> *dst_size = space - s.avail_out;
> - return true;
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> }
>
> -bool uncompress_startswith(const void *src, uint64_t src_size,
> - void **buffer, uint64_t *buffer_size,
> - const void *prefix, uint64_t prefix_len,
> - uint8_t extra) {
> +int decompress_blob_lz4(const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
>
> +#ifdef HAVE_LZ4
> + char* out;
> + uint64_t size;
> + int r;
> +
> + assert(src);
> + assert(src_size > 0);
> + assert(dst);
> + assert(dst_alloc_size);
> + assert(dst_size);
> + assert(*dst_alloc_size == 0 || *dst);
> +
> + if (src_size <= 8)
> + return -EBADMSG;
> +
> + size = le64toh( *(le64_t*)src );
> + if (size > *dst_alloc_size) {
> + out = realloc(*dst, size);
> + if (!out)
> + return -ENOMEM;
> + *dst = out;
> + *dst_alloc_size = size;
> + } else
> + out = *dst;
> +
> + r = LZ4_decompress_safe(src + 8, out, src_size - 8, size);
> + if (r < 0 || (uint64_t) r != size)
> + return -EBADMSG;
> +
> + *dst_size = size;
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +int decompress_blob(int compression,
> + const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max) {
> + if (compression == OBJECT_COMPRESSED_XZ)
> + return decompress_blob_xz(src, src_size,
> + dst, dst_alloc_size, dst_size, dst_max);
> + else if (compression == OBJECT_COMPRESSED_LZ4)
> + return decompress_blob_lz4(src, src_size,
> + dst, dst_alloc_size, dst_size, dst_max);
> + else
> + return -EBADMSG;
> +}
> +
> +
> +int decompress_startswith_xz(const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra) {
> +
> +#ifdef HAVE_XZ
> _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
> lzma_ret ret;
>
> - /* Checks whether the uncompressed blob starts with the
> + /* Checks whether the decompressed blob starts with the
> * mentioned prefix. The byte extra needs to follow the
> * prefix */
>
> @@ -133,7 +234,7 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
>
> ret = lzma_stream_decoder(&s, UINT64_MAX, 0);
> if (ret != LZMA_OK)
> - return false;
> + return -EBADMSG;
>
> if (!(greedy_realloc(buffer, buffer_size, prefix_len + 1, 1)))
> return false;
> @@ -148,25 +249,61 @@ bool uncompress_startswith(const void *src, uint64_t src_size,
> ret = lzma_code(&s, LZMA_FINISH);
>
> if (ret != LZMA_STREAM_END && ret != LZMA_OK)
> - return false;
> + return -EBADMSG;
>
> if (*buffer_size - s.avail_out >= prefix_len + 1)
> return memcmp(*buffer, prefix, prefix_len) == 0 &&
> ((const uint8_t*) *buffer)[prefix_len] == extra;
>
> if (ret == LZMA_STREAM_END)
> - return false;
> + return -EBADMSG;
> +
>
> s.avail_out += *buffer_size;
>
> if (!(greedy_realloc(buffer, buffer_size, *buffer_size * 2, 1)))
> - return false;
> + return -ENOMEM;
>
> s.next_out = *buffer + *buffer_size - s.avail_out;
> }
> +
> + return 1;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +int decompress_startswith_lz4(const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra) {
> +#if HAVE_LZ4
#ifdef
> + return -EPROTONOSUPPORT;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> }
>
> -int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
> +int decompress_startswith(int compression,
> + const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra) {
> + if (compression == OBJECT_COMPRESSED_XZ)
> + return decompress_startswith_xz(src, src_size,
> + buffer, buffer_size,
> + prefix, prefix_len,
> + extra);
> + else if (compression == OBJECT_COMPRESSED_LZ4)
> + return decompress_startswith_lz4(src, src_size,
> + buffer, buffer_size,
> + prefix, prefix_len,
> + extra);
> + else
> + return -EBADMSG;
> +}
> +
> +int compress_stream_xz(int fdf, int fdt, off_t max_bytes) {
> _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
> lzma_ret ret;
>
> @@ -176,7 +313,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
> assert(fdf >= 0);
> assert(fdt >= 0);
>
> - ret = lzma_easy_encoder(&s, preset, LZMA_CHECK_CRC64);
> + ret = lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
> if (ret != LZMA_OK) {
> log_error("Failed to initialize XZ encoder: code %d", ret);
> return -EINVAL;
> @@ -230,7 +367,7 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
> return errno ? -errno : -EIO;
>
> if (ret == LZMA_STREAM_END) {
> - log_debug("Compression finished (%zu -> %zu bytes, %.1f%%)",
> + log_debug("XZ compression finished (%zu -> %zu bytes, %.1f%%)",
> s.total_in, s.total_out,
> (double) s.total_out / s.total_in * 100);
>
> @@ -240,7 +377,91 @@ int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_bytes) {
> }
> }
>
> -int decompress_stream(int fdf, int fdt, off_t max_bytes) {
> +#define LZ4_BUFSIZE (512*1024)
> +
> +int compress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
> +
> +#ifdef HAVE_LZ4
> +
> + _cleanup_free_ char *buf1 = NULL, *buf2 = NULL, *out = NULL;
> + char *buf;
> + LZ4_stream_t lz4_data = {};
> + le32_t header;
> + size_t total_in = 0, total_out = sizeof(header);
> + ssize_t n;
> +
> + assert(fdf >= 0);
> + assert(fdt >= 0);
> +
> + buf1 = malloc(LZ4_BUFSIZE);
> + buf2 = malloc(LZ4_BUFSIZE);
> + out = malloc(LZ4_COMPRESSBOUND(LZ4_BUFSIZE));
> + if (!buf1 || !buf2 || !out)
> + return log_oom();
> +
> + buf = buf1;
> + for (;;) {
> + size_t m;
> + int r;
> +
> + m = LZ4_BUFSIZE;
> + if (max_bytes != -1 && m > (size_t) max_bytes - total_in)
> + m = max_bytes - total_in;
> +
> + n = read(fdf, buf, m);
> + if (n < 0)
> + return -errno;
> + if (n == 0)
> + break;
> +
> + total_in += n;
> +
> + r = LZ4_compress_limitedOutput_continue(&lz4_data, buf, out, n, n);
> + if (r == 0) {
> + log_debug("Compressed size exceeds original, aborting compression.");
> + return -ENOSPC;
> + }
> +
> + header = htole32(r);
> + errno = 0;
> +
> + n = write(fdt, &header, sizeof(header));
> + if (n < 0)
> + return -errno;
> + if (n != sizeof(header))
> + return errno ? -errno : -EIO;
> +
> + n = loop_write(fdt, out, r, false);
> + if (n < 0)
> + return n;
> + if (n != r)
> + return errno ? -errno : -EIO;
> +
> + total_out += sizeof(header) + r;
> +
> + buf = buf == buf1 ? buf2 : buf1;
> + }
> +
> + header = htole32(0);
> + n = write(fdt, &header, sizeof(header));
> + if (n < 0)
> + return -errno;
> + if (n != sizeof(header))
> + return errno ? -errno : -EIO;
> +
> + log_debug("LZ4 compression finished (%zu -> %zu bytes, %.1f%%)",
> + total_in, total_out,
> + (double) total_out / total_in * 100);
> +
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +int decompress_stream_xz(int fdf, int fdt, off_t max_bytes) {
> +
> +#ifdef HAVE_XZ
> _cleanup_(lzma_end) lzma_stream s = LZMA_STREAM_INIT;
> lzma_ret ret;
>
> @@ -302,7 +523,7 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
> return errno ? -errno : -EIO;
>
> if (ret == LZMA_STREAM_END) {
> - log_debug("Decompression finished (%zu -> %zu bytes, %.1f%%)",
> + log_debug("XZ decompression finished (%zu -> %zu bytes, %.1f%%)",
> s.total_in, s.total_out,
> (double) s.total_out / s.total_in * 100);
>
> @@ -310,4 +531,99 @@ int decompress_stream(int fdf, int fdt, off_t max_bytes) {
> }
> }
> }
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +int decompress_stream_lz4(int fdf, int fdt, off_t max_bytes) {
> +
> +#ifdef HAVE_LZ4
> + _cleanup_free_ char *buf = NULL, *out = NULL;
> + size_t buf_size = 0;
> + LZ4_streamDecode_t lz4_data = {};
> + le32_t header;
> + size_t total_in = sizeof(header), total_out = 0;
> +
> + assert(fdf >= 0);
> + assert(fdt >= 0);
> +
> + out = malloc(4*LZ4_BUFSIZE);
> + if (!out)
> + return log_oom();
> +
> + for (;;) {
> + ssize_t n, m;
> + int r;
> +
> + n = read(fdf, &header, sizeof(header));
> + if (n < 0)
> + return -errno;
> + if (n != sizeof(header))
> + return errno ? -errno : -EIO;
> +
> + m = le32toh(header);
> + if (m == 0)
> + break;
> +
> + /* We refuse to use a bigger decompression buffer than
> + * the one used for compression by 4 times. This means
> + * that compression buffer size can be enlarged 4
> + * times. This can be changed, but old binaries might
> + * not accept buffers compressed by newer binaries then.
> + */
> + if (m > LZ4_COMPRESSBOUND(LZ4_BUFSIZE * 4)) {
> + log_error("Compressed stream block too big: %zd bytes", m);
> + return -EBADMSG;
> + }
> +
> + total_in += sizeof(header) + m;
> +
> + if (!GREEDY_REALLOC(buf, buf_size, m))
> + return log_oom();
> +
> + errno = 0;
> + n = loop_read(fdf, buf, m, false);
> + if (n < 0)
> + return n;
> + if (n != m)
> + return errno ? -errno : -EIO;
> +
> + r = LZ4_decompress_safe_continue(&lz4_data, buf, out, m, 4*LZ4_BUFSIZE);
> + if (r <= 0)
> + log_error("LZ4 decompression failed.");
> +
> + total_out += r;
> +
> + if (max_bytes != -1 && total_out > (size_t) max_bytes) {
> + log_debug("Decompressed stream longer than %zd bytes", max_bytes);
> + return -E2BIG;
> + }
> +
> + errno = 0;
> + n = loop_write(fdt, out, r, false);
> + if (n < 0)
> + return n;
> + if (n != r)
> + return errno ? -errno : -EIO;
> + }
> +
> + log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
> + total_in, total_out,
> + (double) total_out / total_in * 100);
> +
> + return 0;
> +#else
> + return -EPROTONOSUPPORT;
> +#endif
> +}
> +
> +int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes) {
> +
> + if (endswith(filename, ".lz4"))
> + return decompress_stream_lz4(fdf, fdt, max_bytes);
> + else if (endswith(filename, ".xz"))
> + return decompress_stream_xz(fdf, fdt, max_bytes);
> + else
> + return -EPROTONOSUPPORT;
> }
> diff --git a/src/journal/compress.h b/src/journal/compress.h
> index f25fe86abd..0810a13c3f 100644
> --- a/src/journal/compress.h
> +++ b/src/journal/compress.h
> @@ -25,15 +25,62 @@
> #include <stdbool.h>
> #include <unistd.h>
>
> -bool compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
> +#include "journal-def.h"
>
> -bool uncompress_blob(const void *src, uint64_t src_size,
> - void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
> +const char* object_compressed_to_string(int compression);
> +int object_compressed_from_string(const char *compression);
>
> -bool uncompress_startswith(const void *src, uint64_t src_size,
> - void **buffer, uint64_t *buffer_size,
> - const void *prefix, uint64_t prefix_len,
> - uint8_t extra);
> +int compress_blob_xz(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
> +int compress_blob_lz4(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size);
>
> -int compress_stream(int fdf, int fdt, uint32_t preset, off_t max_size);
> -int decompress_stream(int fdf, int fdt, off_t max_size);
> +static inline int compress_blob(const void *src, uint64_t src_size, void *dst, uint64_t *dst_size) {
> + int r;
> +#if HAVE_LZ4
#ifdef
> + r = compress_blob_lz4(src, src_size, dst, dst_size);
> + if (r == 0)
> + return OBJECT_COMPRESSED_LZ4;
> +#else
> + r = compress_blob_xz(src, src_size, dst, dst_size);
> + if (r == 0)
> + return OBJECT_COMPRESSED_XZ;
> +#endif
> + return r;
> +}
> +
> +int decompress_blob_xz(const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
> +int decompress_blob_lz4(const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
> +int decompress_blob(int compression,
> + const void *src, uint64_t src_size,
> + void **dst, uint64_t *dst_alloc_size, uint64_t* dst_size, uint64_t dst_max);
> +
> +int decompress_startswith_xz(const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra);
> +int decompress_startswith_lz4(const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra);
> +int decompress_startswith(int compression,
> + const void *src, uint64_t src_size,
> + void **buffer, uint64_t *buffer_size,
> + const void *prefix, uint64_t prefix_len,
> + uint8_t extra);
> +
> +int compress_stream_xz(int fdf, int fdt, off_t max_bytes);
> +int compress_stream_lz4(int fdf, int fdt, off_t max_bytes);
> +
> +int decompress_stream_xz(int fdf, int fdt, off_t max_size);
> +int decompress_stream_lz4(int fdf, int fdt, off_t max_size);
> +
> +#if HAVE_LZ4
#ifdef
> +# define compress_stream compress_stream_lz4
> +# define COMPRESSED_EXT ".lz4"
> +#else
> +# define compress_stream compress_stream_lz4
I think you meant compress_stream_xz
> +# define COMPRESSED_EXT ".xz"
> +#endif
> +
> +int decompress_stream(const char *filename, int fdf, int fdt, off_t max_bytes);
> diff --git a/src/journal/coredump.c b/src/journal/coredump.c
> index 78e89d33cf..e46719706d 100644
> --- a/src/journal/coredump.c
> +++ b/src/journal/coredump.c
> @@ -357,7 +357,7 @@ static int save_external_coredump(
> goto fail;
> }
>
> -#ifdef HAVE_XZ
> +#if defined(HAVE_XZ) || defined(HAVE_LZ4)
> /* If we will remove the coredump anyway, do not compress. */
> if (maybe_remove_external_coredump(NULL, st.st_size) == 0
> && arg_compress) {
> @@ -365,15 +365,15 @@ static int save_external_coredump(
> _cleanup_free_ char *fn_compressed = NULL, *tmp_compressed = NULL;
> _cleanup_close_ int fd_compressed = -1;
>
> - fn_compressed = strappend(fn, ".xz");
> + fn_compressed = strappend(fn, COMPRESSED_EXT);
> if (!fn_compressed) {
> - r = log_oom();
> + log_oom();
> goto uncompressed;
> }
>
> tmp_compressed = tempfn_random(fn_compressed);
> if (!tmp_compressed) {
> - r = log_oom();
> + log_oom();
> goto uncompressed;
> }
>
> @@ -383,7 +383,7 @@ static int save_external_coredump(
> goto uncompressed;
> }
>
> - r = compress_stream(fd, fd_compressed, LZMA_PRESET_DEFAULT, -1);
> + r = compress_stream(fd, fd_compressed, -1);
> if (r < 0) {
> log_error("Failed to compress %s: %s", tmp_compressed, strerror(-r));
> goto fail_compressed;
> diff --git a/src/journal/coredumpctl.c b/src/journal/coredumpctl.c
> index 2158d73771..39f6e65575 100644
> --- a/src/journal/coredumpctl.c
> +++ b/src/journal/coredumpctl.c
> @@ -656,7 +656,7 @@ static int save_core(sd_journal *j, int fd, char **path, bool *unlink_temp) {
> goto error;
> }
>
> - r = decompress_stream(fdf, fd, -1);
> + r = decompress_stream(filename, fdf, fd, -1);
> if (r < 0) {
> log_error("Failed to decompress %s: %s", filename, strerror(-r));
> goto error;
> diff --git a/src/journal/journal-def.h b/src/journal/journal-def.h
> index 7e407a416c..ecfa9a2b16 100644
> --- a/src/journal/journal-def.h
> +++ b/src/journal/journal-def.h
> @@ -66,9 +66,13 @@ enum {
>
> /* Object flags */
> enum {
> - OBJECT_COMPRESSED = 1
> + OBJECT_COMPRESSED_XZ = 1 << 0,
> + OBJECT_COMPRESSED_LZ4 = 1 << 1,
> + _OBJECT_COMPRESSED_MAX
> };
>
> +#define OBJECT_COMPRESSION_MASK (OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4)
> +
> struct ObjectHeader {
> uint8_t type;
> uint8_t flags;
> @@ -155,13 +159,33 @@ enum {
>
> /* Header flags */
> enum {
> - HEADER_INCOMPATIBLE_COMPRESSED = 1
> + HEADER_INCOMPATIBLE_COMPRESSED_XZ = 1 << 0,
> + HEADER_INCOMPATIBLE_COMPRESSED_LZ4 = 1 << 1,
> };
>
> +#define HEADER_INCOMPATIBLE_ANY (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
> +
> +#if defined(HAVE_XZ) && defined(HAVE_LZ4)
> +# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_ANY
> +#elif defined(HAVE_XZ)
> +# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_XZ
> +#elif defined(HAVE_LZ4)
> +# define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_COMPRESSED_LZ4
> +#else
> +# define HEADER_INCOMPATIBLE_SUPPORTED 0
> +#endif
> +
> enum {
> HEADER_COMPATIBLE_SEALED = 1
> };
>
> +#define HEADER_COMPATIBLE_ANY HEADER_COMPATIBLE_SEALED
> +#if HAVE_GCRYPT
#ifdef
> +# define HEADER_COMPATIBLE_SUPPORTED HEADER_COMPATIBLE_SEALED
> +#else
> +# define HEADER_COMPATIBLE_SUPPORTED 0
> +#endif
> +
> #define HEADER_SIGNATURE ((char[]) { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' })
>
> struct Header {
> diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
> index b3b1ffc3c0..b8d375e1d2 100644
> --- a/src/journal/journal-file.c
> +++ b/src/journal/journal-file.c
> @@ -158,21 +158,21 @@ void journal_file_close(JournalFile *f) {
> }
>
> static int journal_file_init_header(JournalFile *f, JournalFile *template) {
> - Header h;
> + Header h = {};
> ssize_t k;
> int r;
>
> assert(f);
>
> - zero(h);
> memcpy(h.signature, HEADER_SIGNATURE, 8);
> h.header_size = htole64(ALIGN64(sizeof(h)));
>
> - h.incompatible_flags =
> - htole32(f->compress ? HEADER_INCOMPATIBLE_COMPRESSED : 0);
> + h.incompatible_flags |= htole32(
> + f->compress_xz * HEADER_INCOMPATIBLE_COMPRESSED_XZ |
> + f->compress_lz4 * HEADER_INCOMPATIBLE_COMPRESSED_LZ4);
>
> - h.compatible_flags =
> - htole32(f->seal ? HEADER_COMPATIBLE_SEALED : 0);
> + h.compatible_flags = htole32(
> + f->seal * HEADER_COMPATIBLE_SEALED);
>
> r = sd_id128_randomize(&h.file_id);
> if (r < 0)
> @@ -222,6 +222,8 @@ static int journal_file_refresh_header(JournalFile *f) {
> }
>
> static int journal_file_verify_header(JournalFile *f) {
> + uint32_t flags;
> +
> assert(f);
>
> if (memcmp(f->header->signature, HEADER_SIGNATURE, 8))
> @@ -229,24 +231,30 @@ static int journal_file_verify_header(JournalFile *f) {
>
> /* In both read and write mode we refuse to open files with
> * incompatible flags we don't know */
> -#ifdef HAVE_XZ
> - if ((le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) != 0)
> + flags = le32toh(f->header->incompatible_flags);
> + if (flags & ~HEADER_INCOMPATIBLE_SUPPORTED) {
> + if (flags & ~HEADER_INCOMPATIBLE_ANY)
> + log_debug("Journal file %s has unknown incompatible flags %"PRIx32,
> + f->path, flags & ~HEADER_INCOMPATIBLE_ANY);
> + flags = (flags & HEADER_INCOMPATIBLE_ANY) & ~HEADER_INCOMPATIBLE_SUPPORTED;
> + if (flags)
> + log_debug("Journal file %s uses incompatible flags %"PRIx32
> + " disabled at compilation time.", f->path, flags);
> return -EPROTONOSUPPORT;
> -#else
> - if (f->header->incompatible_flags != 0)
> - return -EPROTONOSUPPORT;
> -#endif
> + }
>
> /* When open for writing we refuse to open files with
> * compatible flags, too */
> - if (f->writable) {
> -#ifdef HAVE_GCRYPT
> - if ((le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) != 0)
> - return -EPROTONOSUPPORT;
> -#else
> - if (f->header->compatible_flags != 0)
> - return -EPROTONOSUPPORT;
> -#endif
> + flags = le32toh(f->header->compatible_flags);
> + if (f->writable && (flags & ~HEADER_COMPATIBLE_SUPPORTED)) {
> + if (flags & ~HEADER_COMPATIBLE_ANY)
> + log_debug("Journal file %s has unknown compatible flags %"PRIx32,
> + f->path, flags & ~HEADER_COMPATIBLE_ANY);
> + flags = (flags & HEADER_COMPATIBLE_ANY) & ~HEADER_COMPATIBLE_SUPPORTED;
> + if (flags)
> + log_debug("Journal file %s uses compatible flags %"PRIx32
> + " disabled at compilation time.", f->path, flags);
> + return -EPROTONOSUPPORT;
> }
>
> if (f->header->state >= _STATE_MAX)
> @@ -302,7 +310,8 @@ static int journal_file_verify_header(JournalFile *f) {
> }
> }
>
> - f->compress = JOURNAL_HEADER_COMPRESSED(f->header);
> + f->compress_xz = JOURNAL_HEADER_COMPRESSED_XZ(f->header);
> + f->compress_lz4 = JOURNAL_HEADER_COMPRESSED_LZ4(f->header);
>
> f->seal = JOURNAL_HEADER_SEALED(f->header);
>
> @@ -809,8 +818,7 @@ int journal_file_find_data_object_with_hash(
> if (le64toh(o->data.hash) != hash)
> goto next;
>
> - if (o->object.flags & OBJECT_COMPRESSED) {
> -#ifdef HAVE_XZ
> + if (o->object.flags & OBJECT_COMPRESSION_MASK) {
> uint64_t l, rsize;
>
> l = le64toh(o->object.size);
> @@ -819,8 +827,10 @@ int journal_file_find_data_object_with_hash(
>
> l -= offsetof(Object, data.payload);
>
> - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0))
> - return -EBADMSG;
> + r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
> + o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, 0);
> + if (r < 0)
> + return r;
>
> if (rsize == size &&
> memcmp(f->compress_buffer, data, size) == 0) {
> @@ -833,9 +843,6 @@ int journal_file_find_data_object_with_hash(
>
> return 1;
> }
> -#else
> - return -EPROTONOSUPPORT;
> -#endif
>
> } else if (le64toh(o->object.size) == osize &&
> memcmp(o->data.payload, data, size) == 0) {
> @@ -943,8 +950,7 @@ static int journal_file_append_data(
> uint64_t hash, p;
> uint64_t osize;
> Object *o;
> - int r;
> - bool compressed = false;
> + int r, compression = 0;
> const void *eq;
>
> assert(f);
> @@ -974,22 +980,23 @@ static int journal_file_append_data(
> o->data.hash = htole64(hash);
>
> #ifdef HAVE_XZ
> - if (f->compress &&
> + if (f->compress_xz &&
> size >= COMPRESSION_SIZE_THRESHOLD) {
> uint64_t rsize;
>
> - compressed = compress_blob(data, size, o->data.payload, &rsize);
> + compression = compress_blob(data, size, o->data.payload, &rsize);
>
> - if (compressed) {
> + if (compression) {
> o->object.size = htole64(offsetof(Object, data.payload) + rsize);
> - o->object.flags |= OBJECT_COMPRESSED;
> + o->object.flags |= compression;
>
> - log_debug("Compressed data object %"PRIu64" -> %"PRIu64, size, rsize);
> + log_debug("Compressed data object %"PRIu64" -> %"PRIu64" using %s",
> + size, rsize, object_compressed_to_string(compression));
> }
> }
> #endif
>
> - if (!compressed && size > 0)
> + if (!compression && size > 0)
> memcpy(o->data.payload, data, size);
>
> r = journal_file_link_data(f, o, p, hash);
> @@ -2332,8 +2339,9 @@ void journal_file_dump(JournalFile *f) {
> break;
> }
>
> - if (o->object.flags & OBJECT_COMPRESSED)
> - printf("Flags: COMPRESSED\n");
> + if (o->object.flags & OBJECT_COMPRESSION_MASK)
> + printf("Flags: %s\n",
> + object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK));
>
> if (p == le64toh(f->header->tail_object_offset))
> p = 0;
> @@ -2370,7 +2378,7 @@ void journal_file_print_header(JournalFile *f) {
> "Sequential Number ID: %s\n"
> "State: %s\n"
> "Compatible Flags:%s%s\n"
> - "Incompatible Flags:%s%s\n"
> + "Incompatible Flags:%s%s%s\n"
> "Header size: %"PRIu64"\n"
> "Arena size: %"PRIu64"\n"
> "Data Hash Table Size: %"PRIu64"\n"
> @@ -2392,9 +2400,10 @@ void journal_file_print_header(JournalFile *f) {
> f->header->state == STATE_ONLINE ? "ONLINE" :
> f->header->state == STATE_ARCHIVED ? "ARCHIVED" : "UNKNOWN",
> JOURNAL_HEADER_SEALED(f->header) ? " SEALED" : "",
> - (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_SEALED) ? " ???" : "",
> - JOURNAL_HEADER_COMPRESSED(f->header) ? " COMPRESSED" : "",
> - (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_COMPRESSED) ? " ???" : "",
> + (le32toh(f->header->compatible_flags) & ~HEADER_COMPATIBLE_ANY) ? " ???" : "",
> + JOURNAL_HEADER_COMPRESSED_XZ(f->header) ? " COMPRESSED-XZ" : "",
> + JOURNAL_HEADER_COMPRESSED_LZ4(f->header) ? " COMPRESSED-LZ4" : "",
> + (le32toh(f->header->incompatible_flags) & ~HEADER_INCOMPATIBLE_ANY) ? " ???" : "",
> le64toh(f->header->header_size),
> le64toh(f->header->arena_size),
> le64toh(f->header->data_hash_table_size) / sizeof(HashItem),
> @@ -2468,7 +2477,7 @@ int journal_file_open(
> f->prot = prot_from_flags(flags);
> f->writable = (flags & O_ACCMODE) != O_RDONLY;
> #ifdef HAVE_XZ
> - f->compress = compress;
> + f->compress_xz = compress;
> #endif
> #ifdef HAVE_GCRYPT
> f->seal = seal;
> @@ -2760,11 +2769,11 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
> if ((uint64_t) t != l)
> return -E2BIG;
>
> - if (o->object.flags & OBJECT_COMPRESSED) {
> + if (o->object.flags & OBJECT_COMPRESSED_XZ) {
> #ifdef HAVE_XZ
> uint64_t rsize;
>
> - if (!uncompress_blob(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
> + if (!decompress_blob_xz(o->data.payload, l, &from->compress_buffer, &from->compress_buffer_size, &rsize, 0))
> return -EBADMSG;
>
> data = from->compress_buffer;
> diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
> index 50dbe29cc1..b0c28b5e83 100644
> --- a/src/journal/journal-file.h
> +++ b/src/journal/journal-file.h
> @@ -56,7 +56,8 @@ typedef struct JournalFile {
> int flags;
> int prot;
> bool writable:1;
> - bool compress:1;
> + bool compress_xz:1;
> + bool compress_lz4:1;
> bool seal:1;
>
> bool tail_entry_monotonic_valid:1;
> @@ -153,8 +154,11 @@ static inline bool VALID_EPOCH(uint64_t u) {
> #define JOURNAL_HEADER_SEALED(h) \
> (!!(le32toh((h)->compatible_flags) & HEADER_COMPATIBLE_SEALED))
>
> -#define JOURNAL_HEADER_COMPRESSED(h) \
> - (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED))
> +#define JOURNAL_HEADER_COMPRESSED_XZ(h) \
> + (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_XZ))
> +
> +#define JOURNAL_HEADER_COMPRESSED_LZ4(h) \
> + (!!(le32toh((h)->incompatible_flags) & HEADER_INCOMPATIBLE_COMPRESSED_LZ4))
>
> int journal_file_move_to_object(JournalFile *f, int type, uint64_t offset, Object **ret);
>
> diff --git a/src/journal/journal-verify.c b/src/journal/journal-verify.c
> index 31bae5a8f8..c063d12210 100644
> --- a/src/journal/journal-verify.c
> +++ b/src/journal/journal-verify.c
> @@ -45,7 +45,7 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
> * possible field values. It does not follow any references to
> * other objects. */
>
> - if ((o->object.flags & OBJECT_COMPRESSED) &&
> + if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
> o->object.type != OBJECT_DATA)
> return -EBADMSG;
>
> @@ -72,22 +72,38 @@ static int journal_file_object_verify(JournalFile *f, uint64_t offset, Object *o
>
> h1 = le64toh(o->data.hash);
>
> - if (o->object.flags & OBJECT_COMPRESSED) {
> + if (o->object.flags & OBJECT_COMPRESSED_XZ) {
> #ifdef HAVE_XZ
> - void *b = NULL;
> + _cleanup_free_ void *b = NULL;
> uint64_t alloc = 0, b_size;
>
> - if (!uncompress_blob(o->data.payload,
> - le64toh(o->object.size) - offsetof(Object, data.payload),
> - &b, &alloc, &b_size, 0)) {
> - log_error(OFSfmt": uncompression failed", offset);
> + if (!decompress_blob_xz(o->data.payload,
> + le64toh(o->object.size) - offsetof(Object, data.payload),
> + &b, &alloc, &b_size, 0)) {
> + log_error(OFSfmt": XZ decompression failed", offset);
> return -EBADMSG;
> }
>
> h2 = hash64(b, b_size);
> - free(b);
> #else
> - log_error("Compression is not supported");
> + log_error("XZ compression is not supported");
> + return -EPROTONOSUPPORT;
> +#endif
> + } else if (o->object.flags & OBJECT_COMPRESSED_LZ4) {
> +#ifdef HAVE_XZ
#ifdef HAVE_LZ4
> + _cleanup_free_ void *b = NULL;
> + uint64_t alloc = 0, b_size;
> +
> + if (!decompress_blob_xz(o->data.payload,
decompress_blob_lz4
> + le64toh(o->object.size) - offsetof(Object, data.payload),
> + &b, &alloc, &b_size, 0)) {
> + log_error(OFSfmt": LZ4 decompression failed", offset);
> + return -EBADMSG;
> + }
> +
> + h2 = hash64(b, b_size);
> +#else
> + log_error("XZ compression is not supported");
-XZ +LZ4
> return -EPROTONOSUPPORT;
> #endif
> } else
> @@ -875,8 +891,21 @@ int journal_file_verify(
> goto fail;
> }
>
> - if ((o->object.flags & OBJECT_COMPRESSED) && !JOURNAL_HEADER_COMPRESSED(f->header)) {
> - log_error("Compressed object in file without compression at "OFSfmt, p);
> + if ((o->object.flags & OBJECT_COMPRESSED_XZ) &&
> + (o->object.flags & OBJECT_COMPRESSED_LZ4)) {
> + log_error("Objected with double compression at "OFSfmt, p);
> + r = -EBADMSG;
> + goto fail;
> + }
> +
> + if ((o->object.flags & OBJECT_COMPRESSED_XZ) && !JOURNAL_HEADER_COMPRESSED_XZ(f->header)) {
> + log_error("XZ compressed object in file without XZ compression at "OFSfmt, p);
> + r = -EBADMSG;
> + goto fail;
> + }
> +
> + if ((o->object.flags & OBJECT_COMPRESSED_LZ4) && !JOURNAL_HEADER_COMPRESSED_LZ4(f->header)) {
> + log_error("LZ4 compressed object in file without LZ4 compression at "OFSfmt, p);
> r = -EBADMSG;
> goto fail;
> }
> diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
> index ca805f83fe..96ba2bd11e 100644
> --- a/src/journal/sd-journal.c
> +++ b/src/journal/sd-journal.c
> @@ -1995,28 +1995,24 @@ _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **
>
> l = le64toh(o->object.size) - offsetof(Object, data.payload);
>
> - if (o->object.flags & OBJECT_COMPRESSED) {
> + if ((o->object.flags & OBJECT_COMPRESSION_MASK) &&
> + decompress_startswith(o->object.flags & OBJECT_COMPRESSION_MASK,
> + o->data.payload, l,
> + &f->compress_buffer, &f->compress_buffer_size,
> + field, field_length, '=')) {
>
> -#ifdef HAVE_XZ
> - if (uncompress_startswith(o->data.payload, l,
> - &f->compress_buffer, &f->compress_buffer_size,
> - field, field_length, '=')) {
> -
> - uint64_t rsize;
> + uint64_t rsize;
>
> - if (!uncompress_blob(o->data.payload, l,
> - &f->compress_buffer, &f->compress_buffer_size, &rsize,
> - j->data_threshold))
> - return -EBADMSG;
> + r = decompress_blob_xz(o->data.payload, l,
> + &f->compress_buffer, &f->compress_buffer_size, &rsize,
> + j->data_threshold);
> + if (r < 0)
> + return r;
>
> - *data = f->compress_buffer;
> - *size = (size_t) rsize;
> + *data = f->compress_buffer;
> + *size = (size_t) rsize;
>
> - return 0;
> - }
> -#else
> - return -EPROTONOSUPPORT;
> -#endif
> + return 0;
>
> } else if (l >= field_length+1 &&
> memcmp(o->data.payload, field, field_length) == 0 &&
> @@ -2052,11 +2048,11 @@ static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **da
> if ((uint64_t) t != l)
> return -E2BIG;
>
> - if (o->object.flags & OBJECT_COMPRESSED) {
> + if (o->object.flags & OBJECT_COMPRESSED_XZ) {
> #ifdef HAVE_XZ
> uint64_t rsize;
>
> - if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
> + if (!decompress_blob_xz(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
> return -EBADMSG;
>
> *data = f->compress_buffer;
> diff --git a/src/journal/test-compress.c b/src/journal/test-compress.c
> index 0806145d4b..24171463a2 100644
> --- a/src/journal/test-compress.c
> +++ b/src/journal/test-compress.c
> @@ -21,54 +21,110 @@
> #include "util.h"
> #include "macro.h"
>
> -static void test_compress_uncompress(void) {
> +#ifdef HAVE_XZ
> +# define XZ_OK 0
> +#else
> +# define XZ_OK -EPROTONOSUPPORT
> +#endif
> +
> +#ifdef HAVE_XZ
#ifdef HAVE_LZ4
> +# define LZ4_OK 0
> +#else
> +# define LZ4_OK -EPROTONOSUPPORT
> +#endif
> +
> +static void test_compress_decompress(void) {
> char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
> "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
> char compressed[512];
> uint64_t csize = 512;
> uint64_t usize = 0;
> - _cleanup_free_ char *uncompressed = NULL;
> -
> - assert_se(compress_blob(text, sizeof(text), compressed, &csize));
> - assert_se(uncompress_blob(compressed,
> - csize,
> - (void **) &uncompressed,
> - &usize, &csize, 0));
> - assert_se(uncompressed);
> - assert_se(streq(uncompressed, text));
> - assert_se(!uncompress_blob("garbage",
> - 7,
> - (void **) &uncompressed,
> - &usize, &csize, 0));
> + _cleanup_free_ char *decompressed = NULL, *decompressed2 = NULL;
> + int r;
> +
> + r = compress_blob_xz(text, sizeof(text), compressed, &csize);
> + assert(r == XZ_OK);
> + r = decompress_blob_xz(compressed,
> + csize,
> + (void **) &decompressed,
> + &usize, &csize, 0);
> + assert(r == XZ_OK);
> +#ifdef HAVE_XZ
> + assert_se(decompressed);
> + assert_se(streq(decompressed, text));
> +#endif
> + r = decompress_blob_xz("garbage",
> + 7,
> + (void **) &decompressed,
> + &usize, &csize, 0);
> + assert(r < 0);
> +
> +
> + r = compress_blob_lz4(text, sizeof(text), compressed, &csize);
> + assert(r == LZ4_OK);
> + r = decompress_blob_lz4(compressed,
> + csize,
> + (void **) &decompressed2,
> + &usize, &csize, 0);
> + assert(r == LZ4_OK);
> +#ifdef HAVE_LZ4
> + assert_se(decompressed2);
> + assert_se(streq(decompressed2, text));
> +#endif
> +
> + r = decompress_blob_lz4("garbage",
> + 7,
> + (void **) &decompressed2,
> + &usize, &csize, 0);
> + assert(r < 0);
> +
> + r = decompress_blob_lz4("00000000\1g",
> + 9,
> + (void **) &decompressed2,
> + &usize, &csize, 0);
> + assert(r < 0);
> }
>
> -static void test_uncompress_startswith(void) {
> +static void test_decompress_startswith(void) {
> char text[] = "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
> "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
> char compressed[512];
> uint64_t csize = 512;
> uint64_t usize = 0;
> - _cleanup_free_ char *uncompressed = NULL;
> -
> - assert_se(compress_blob(text, sizeof(text), compressed, &csize));
> - assert_se(uncompress_startswith(compressed,
> - csize,
> - (void **) &uncompressed,
> - &usize,
> - "foofoofoofoo", 12, ' '));
> - assert_se(!uncompress_startswith(compressed,
> - csize,
> - (void **) &uncompressed,
> - &usize,
> - "foofoofoofoo", 12, 'w'));
> - assert_se(!uncompress_startswith(compressed,
> - csize,
> - (void **) &uncompressed,
> - &usize,
> - "barbarbar", 9, ' '));
> + _cleanup_free_ char *decompressed = NULL;
> +
> + assert_se(compress_blob_xz(text, sizeof(text), compressed, &csize) == 0);
> + assert_se(decompress_startswith_xz(compressed,
> + csize,
> + (void **) &decompressed,
> + &usize,
> + "foofoofoofoo", 12, ' ') > 0);
> + assert_se(decompress_startswith_xz(compressed,
> + csize,
> + (void **) &decompressed,
> + &usize,
> + "foofoofoofoo", 12, 'w') == 0);
> + assert_se(decompress_startswith_xz(compressed,
> + csize,
> + (void **) &decompressed,
> + &usize,
> + "barbarbar", 9, ' ') == 0);
> + assert_se(decompress_startswith_xz(compressed,
> + csize,
> + (void **) &decompressed,
> + &usize,
> + "foofoofoofoo", 12, ' ') > 0);
> }
>
> -static void test_compress_stream(const char *srcfile) {
> +typedef int (compress_stream_t)(int fdf, int fdt, off_t max_bytes);
> +typedef int (decompress_stream_t)(int fdf, int fdt, off_t max_size);
> +
> +static void test_compress_stream(const char* type,
> + const char* cat,
> + compress_stream_t compress,
> + decompress_stream_t decompress,
> + const char *srcfile) {
> +
> _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
> char pattern[] = "/tmp/systemd-test.xz.XXXXXX",
> pattern2[] = "/tmp/systemd-test.xz.XXXXXX";
> @@ -76,6 +132,8 @@ static void test_compress_stream(const char *srcfile) {
> _cleanup_free_ char *cmd, *cmd2;
> struct stat st = {};
>
> + log_debug("/* testing %s compression */", type);
> +
> log_debug("/* create source from %s */", srcfile);
>
> assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
> @@ -84,11 +142,13 @@ static void test_compress_stream(const char *srcfile) {
>
> assert_se((dst = mkostemp_safe(pattern, O_RDWR|O_CLOEXEC)) >= 0);
>
> - r = compress_stream(src, dst, 1, -1);
> + r = compress(src, dst, -1);
> + log_debug("compress -> %s", strerror(-r));
> assert(r == 0);
>
> - assert_se(asprintf(&cmd, "xzcat %s | diff %s -", pattern, srcfile) > 0);
> - assert_se(system(cmd) == 0);
> + assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
> + if (system(cmd) == 0)
> + log_warning("External decompression failed.");
>
> log_debug("/* test decompression */");
>
> @@ -97,7 +157,7 @@ static void test_compress_stream(const char *srcfile) {
> assert_se(stat(srcfile, &st) == 0);
>
> assert_se(lseek(dst, 0, SEEK_SET) == 0);
> - r = decompress_stream(dst, dst2, st.st_size);
> + r = decompress(dst, dst2, st.st_size);
> assert(r == 0);
>
> assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
> @@ -106,12 +166,12 @@ static void test_compress_stream(const char *srcfile) {
> log_debug("/* test faulty decompression */");
>
> assert_se(lseek(dst, 1, SEEK_SET) == 1);
> - r = decompress_stream(dst, dst2, st.st_size);
> + r = decompress(dst, dst2, st.st_size);
> assert(r == -EBADMSG);
>
> assert_se(lseek(dst, 0, SEEK_SET) == 0);
> assert_se(lseek(dst2, 0, SEEK_SET) == 0);
> - r = decompress_stream(dst, dst2, st.st_size - 1);
> + r = decompress(dst, dst2, st.st_size - 1);
> assert(r == -E2BIG);
>
> assert_se(unlink(pattern) == 0);
> @@ -122,9 +182,21 @@ int main(int argc, char *argv[]) {
>
> log_set_max_level(LOG_DEBUG);
>
> - test_compress_uncompress();
> - test_uncompress_startswith();
> - test_compress_stream(argv[0]);
> + test_compress_decompress();
> + test_decompress_startswith();
> +
> +#ifdef HAVE_XZ
> + test_compress_stream("XZ", "xzcat",
> + compress_stream_xz, decompress_stream_xz, argv[0]);
> +#else
> + log_info("/* XZ test skipped */");
> +#endif
> +#ifdef HAVE_LZ4
> + test_compress_stream("LZ4", "lz4cat",
> + compress_stream_lz4, decompress_stream_lz4, argv[0]);
> +#else
> + log_info("/* LZ4 test skipped */");
> +#endif
>
> return 0;
> }
> --
> 2.0.0
>
> _______________________________________________
> systemd-devel mailing list
> systemd-devel at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/systemd-devel
More information about the systemd-devel
mailing list