[systemd-commits] 3 commits - .gitignore Makefile.am TODO configure.ac src/import
Lennart Poettering
lennart at kemper.freedesktop.org
Fri Jan 16 11:13:01 PST 2015
.gitignore | 1
Makefile.am | 32 ++++
TODO | 2
configure.ac | 13 +
src/import/import-raw.c | 235 ++++++++++++++++++++++++++++----
src/import/import.c | 48 ++++++
src/import/qcow2-util.c | 347 ++++++++++++++++++++++++++++++++++++++++++++++++
src/import/qcow2-util.h | 25 +++
src/import/test-qcow2.c | 55 +++++++
9 files changed, 724 insertions(+), 34 deletions(-)
New commits:
commit 2a2054a796e396d1e6e679c70ac273f76ccfb39a
Author: Lennart Poettering <lennart at poettering.net>
Date: Fri Jan 16 20:09:23 2015 +0100
update TODO
diff --git a/TODO b/TODO
index 5736a5b..d443209 100644
--- a/TODO
+++ b/TODO
@@ -39,6 +39,8 @@ Release 219 preparations:
Features:
+* import: when pulling raw images, detect NUL blocks, and create sparse files
+
* import: support import from local files, and export to local files
* import: add "pull-tar" support, for downloading/verifying tarballs
commit edce2aed3aa93b84f7b4c70412bdb665da2977b0
Author: Lennart Poettering <lennart at poettering.net>
Date: Fri Jan 16 20:07:25 2015 +0100
import: support importing qcow2 images
With this change the import tool will now unpack qcow2 images into
normal raw disk images, suitable for usage with nspawn.
This allows has the benefit of also allowing importing Ubuntu Cloud
images for usage with nspawn.
diff --git a/.gitignore b/.gitignore
index caef831..d4704fe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -227,6 +227,7 @@
/test-pppoe
/test-prioq
/test-pty
+/test-qcow2
/test-ratelimit
/test-replace-var
/test-resolve
diff --git a/Makefile.am b/Makefile.am
index ee73598..f91c786 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5233,19 +5233,43 @@ systemd_import_SOURCES = \
src/import/curl-util.c \
src/import/curl-util.h \
src/import/aufs-util.c \
- src/import/aufs-util.h
+ src/import/aufs-util.h \
+ src/import/qcow2-util.c \
+ src/import/qcow2-util.h
systemd_import_CFLAGS = \
$(AM_CFLAGS) \
$(LIBCURL_CFLAGS) \
- $(XZ_CFLAGS)
+ $(XZ_CFLAGS) \
+ $(ZLIB_CFLAGS)
systemd_import_LDADD = \
libsystemd-internal.la \
libsystemd-label.la \
libsystemd-shared.la \
$(LIBCURL_LIBS) \
- $(XZ_LIBS)
+ $(XZ_LIBS) \
+ $(ZLIB_LIBS)
+
+endif
+
+if HAVE_ZLIB
+manual_tests += \
+ test-qcow2
+
+test_qcow2_SOURCES = \
+ src/import/test-qcow2.c \
+ src/import/qcow2-util.c \
+ src/import/qcow2-util.h
+
+test_qcow2_CFLAGS = \
+ $(AM_CFLAGS) \
+ $(ZLIB_CFLAGS)
+
+test_qcow2_LDADD = \
+ libsystemd-internal.la \
+ libsystemd-shared.la \
+ $(ZLIB_LIBS)
endif
endif
diff --git a/configure.ac b/configure.ac
index 5057f8e..6d510df 100644
--- a/configure.ac
+++ b/configure.ac
@@ -564,6 +564,18 @@ fi
AM_CONDITIONAL(HAVE_XZ, [test "$have_xz" = "yes"])
# ------------------------------------------------------------------------------
+have_zlib=no
+AC_ARG_ENABLE(zlib, AS_HELP_STRING([--disable-zlib], [Disable optional ZLIB support]))
+if test "x$enable_zlib" != "xno"; then
+ PKG_CHECK_MODULES(ZLIB, [ zlib ],
+ [AC_DEFINE(HAVE_ZLIB, 1, [Define if ZLIB is available]) have_zlib=yes])
+ if test "x$have_zlib" = xno -a "x$enable_zlib" = xyes; then
+ AC_MSG_ERROR([*** ZLIB support requested but libraries not found])
+ fi
+fi
+AM_CONDITIONAL(HAVE_ZLIB, [test "$have_zlib" = "yes"])
+
+# ------------------------------------------------------------------------------
have_lz4=no
AC_ARG_ENABLE(lz4, AS_HELP_STRING([--enable-lz4], [Enable optional LZ4 support]))
AS_IF([test "x$enable_lz4" = "xyes"], [
@@ -1410,6 +1422,7 @@ AC_MSG_RESULT([
SELinux: ${have_selinux}
SECCOMP: ${have_seccomp}
SMACK: ${have_smack}
+ ZLIB: ${have_zlib}
XZ: ${have_xz}
LZ4: ${have_lz4}
ACL: ${have_acl}
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index f1b36cb..c15765d 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -27,6 +27,7 @@
#include "hashmap.h"
#include "utf8.h"
#include "curl-util.h"
+#include "qcow2-util.h"
#include "import-raw.h"
#include "strv.h"
#include "copy.h"
@@ -235,6 +236,49 @@ finish:
raw_import_finish(f->import, r);
}
+static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
+ _cleanup_close_ int converted_fd = -1;
+ _cleanup_free_ char *t = NULL;
+ int r;
+
+ assert(f);
+ assert(f->disk_fd);
+ assert(f->temp_path);
+
+ r = qcow2_detect(f->disk_fd);
+ if (r < 0)
+ return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
+ if (r == 0)
+ return 0;
+
+ /* This is a QCOW2 image, let's convert it */
+ r = tempfn_random(f->final_path, &t);
+ if (r < 0)
+ return log_oom();
+
+ converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0644);
+ if (converted_fd < 0)
+ return log_error_errno(errno, "Failed to create %s: %m", t);
+
+ r = qcow2_convert(f->disk_fd, converted_fd);
+ if (r < 0) {
+ unlink(t);
+ return log_error_errno(r, "Failed to convert qcow2 image: %m");
+ }
+
+ unlink(f->temp_path);
+ free(f->temp_path);
+
+ f->temp_path = t;
+ t = NULL;
+
+ safe_close(f->disk_fd);
+ f->disk_fd = converted_fd;
+ converted_fd = -1;
+
+ return 1;
+}
+
static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
RawImportFile *f = NULL;
struct stat st;
@@ -288,6 +332,10 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result
goto fail;
}
+ r = raw_import_maybe_convert_qcow2(f);
+ if (r < 0)
+ goto fail;
+
if (f->etag)
(void) fsetxattr(f->disk_fd, "user.source_etag", f->etag, strlen(f->etag), 0);
if (f->url)
diff --git a/src/import/import.c b/src/import/import.c
index e457adf..af8d0ec 100644
--- a/src/import/import.c
+++ b/src/import/import.c
@@ -45,10 +45,48 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
sd_event_exit(event, error);
}
+static int strip_raw_suffixes(const char *p, char **ret) {
+ static const char suffixes[] =
+ ".xz\0"
+ ".raw\0"
+ ".qcow2\0"
+ ".img\0";
+
+ _cleanup_free_ char *q = NULL;
+
+ q = strdup(p);
+ if (!q)
+ return -ENOMEM;
+
+ for (;;) {
+ const char *sfx;
+ bool changed = false;
+
+ NULSTR_FOREACH(sfx, suffixes) {
+ char *e;
+
+ e = endswith(q, sfx);
+ if (e) {
+ *e = 0;
+ changed = true;
+ }
+ }
+
+ if (!changed)
+ break;
+ }
+
+ *ret = q;
+ q = NULL;
+
+ return 0;
+}
+
static int pull_raw(int argc, char *argv[], void *userdata) {
_cleanup_(raw_import_unrefp) RawImport *import = NULL;
_cleanup_event_unref_ sd_event *event = NULL;
- const char *url, *local, *suffix;
+ const char *url, *local;
+ _cleanup_free_ char *l = NULL;
int r;
url = argv[1];
@@ -79,13 +117,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
if (local) {
const char *p;
- suffix = endswith(local, ".raw.xz");
- if (!suffix)
- suffix = endswith(local, ".raw");
- if (!suffix)
- suffix = endswith(local, ".xz");
- if (suffix)
- local = strndupa(local, suffix - local);
+ r = strip_raw_suffixes(local, &l);
+ if (r < 0)
+ return log_oom();
+
+ local = l;
if (!machine_name_is_valid(local)) {
log_error("Local image name '%s' is not valid.", local);
diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c
new file mode 100644
index 0000000..c84c6aa
--- /dev/null
+++ b/src/import/qcow2-util.c
@@ -0,0 +1,347 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+ This file is part of systemd.
+
+ Copyright 2015 Lennart Poettering
+
+ systemd is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1 of the License, or
+ (at your option) any later version.
+
+ systemd is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include <zlib.h>
+
+#include "util.h"
+#include "sparse-endian.h"
+#include "qcow2-util.h"
+
+#define QCOW2_MAGIC 0x514649fb
+
+#define QCOW2_COPIED (1ULL << 63)
+#define QCOW2_COMPRESSED (1ULL << 62)
+#define QCOW2_ZERO (1ULL << 0)
+
+typedef struct _packed_ Header {
+ be32_t magic;
+ be32_t version;
+
+ be64_t backing_file_offset;
+ be32_t backing_file_size;
+
+ be32_t cluster_bits;
+ be64_t size;
+ be32_t crypt_method;
+
+ be32_t l1_size;
+ be64_t l1_table_offset;
+
+ be64_t refcount_table_offset;
+ be32_t refcount_table_clusters;
+
+ be32_t nb_snapshots;
+ be64_t snapshots_offset;
+
+ /* The remainder is only present on QCOW3 */
+ be64_t incompatible_features;
+ be64_t compatible_features;
+ be64_t autoclear_features;
+
+ be32_t refcount_order;
+ be32_t header_length;
+} Header;
+
+#define HEADER_MAGIC(header) be32toh((header)->magic)
+#define HEADER_VERSION(header) be32toh((header)->version)
+#define HEADER_CLUSTER_BITS(header) be32toh((header)->cluster_bits)
+#define HEADER_CLUSTER_SIZE(header) (1ULL << HEADER_CLUSTER_BITS(header))
+#define HEADER_L2_BITS(header) (HEADER_CLUSTER_BITS(header) - 3)
+#define HEADER_SIZE(header) be64toh((header)->size)
+#define HEADER_CRYPT_METHOD(header) be32toh((header)->crypt_method)
+#define HEADER_L1_SIZE(header) be32toh((header)->l1_size)
+#define HEADER_L2_SIZE(header) (HEADER_CLUSTER_SIZE(header)/sizeof(uint64_t))
+#define HEADER_L1_TABLE_OFFSET(header) be64toh((header)->l1_table_offset)
+
+static uint32_t HEADER_HEADER_LENGTH(const Header *h) {
+ if (HEADER_VERSION(h) < 3)
+ return offsetof(Header, incompatible_features);
+
+ return be32toh(h->header_length);
+}
+
+static int copy_cluster(
+ int sfd, uint64_t soffset,
+ int dfd, uint64_t doffset,
+ uint64_t cluster_size,
+ void *buffer) {
+
+ ssize_t l;
+
+ l = pread(sfd, buffer, cluster_size, soffset);
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != cluster_size)
+ return -EIO;
+
+ l = pwrite(dfd, buffer, cluster_size, doffset);
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != cluster_size)
+ return -EIO;
+
+ return 0;
+}
+
+static int decompress_cluster(
+ int sfd, uint64_t soffset,
+ int dfd, uint64_t doffset,
+ uint64_t compressed_size,
+ uint64_t cluster_size,
+ void *buffer1,
+ void *buffer2) {
+
+ _cleanup_free_ void *large_buffer = NULL;
+ z_stream s = {};
+ uint64_t sz;
+ ssize_t l;
+ int r;
+
+ if (compressed_size > cluster_size) {
+ /* The usual cluster buffer doesn't suffice, let's
+ * allocate a larger one, temporarily */
+
+ large_buffer = malloc(compressed_size);
+ if (!large_buffer)
+ return -ENOMEM;
+
+ buffer1 = large_buffer;
+ }
+
+ l = pread(sfd, buffer1, compressed_size, soffset);
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != compressed_size)
+ return -EIO;
+
+ s.next_in = buffer1;
+ s.avail_in = compressed_size;
+ s.next_out = buffer2;
+ s.avail_out = cluster_size;
+
+ r = inflateInit2(&s, -12);
+ if (r != Z_OK)
+ return -EIO;
+
+ r = inflate(&s, Z_FINISH);
+ sz = (uint8_t*) s.next_out - (uint8_t*) buffer2;
+ inflateEnd(&s);
+ if (r != Z_STREAM_END || sz != cluster_size)
+ return -EIO;
+
+ l = pwrite(dfd, buffer2, cluster_size, doffset);
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != cluster_size)
+ return -EIO;
+
+ return 0;
+}
+
+static int normalize_offset(
+ const Header *header,
+ uint64_t p,
+ uint64_t *ret,
+ bool *compressed,
+ uint64_t *compressed_size) {
+
+ uint64_t q;
+
+ q = be64toh(p);
+
+ if (q & QCOW2_COMPRESSED) {
+ uint64_t sz, csize_shift, csize_mask;
+
+ if (!compressed)
+ return -ENOTSUP;
+
+ csize_shift = 64 - 2 - (HEADER_CLUSTER_BITS(header) - 8);
+ csize_mask = (1ULL << (HEADER_CLUSTER_BITS(header) - 8)) - 1;
+ sz = (((q >> csize_shift) & csize_mask) + 1) * 512 - (q & 511);
+ q &= ((1ULL << csize_shift) - 1);
+
+ if (compressed_size)
+ *compressed_size = sz;
+
+ *compressed = true;
+
+ } else {
+ if (compressed) {
+ *compressed = false;
+ *compressed_size = 0;
+ }
+
+ if (q & QCOW2_ZERO) {
+ /* We make no distinction between zero blocks and holes */
+ *ret = 0;
+ return 0;
+ }
+
+ q &= ~QCOW2_COPIED;
+ }
+
+ *ret = q;
+ return q > 0; /* returns positive if not a hole */
+}
+
+static int verify_header(const Header *header) {
+ assert(header);
+
+ if (HEADER_MAGIC(header) != QCOW2_MAGIC)
+ return -EBADMSG;
+
+ if (HEADER_VERSION(header) != 2 &&
+ HEADER_VERSION(header) != 3)
+ return -ENOTSUP;
+
+ if (HEADER_CRYPT_METHOD(header) != 0)
+ return -ENOTSUP;
+
+ if (HEADER_CLUSTER_BITS(header) < 9) /* 512K */
+ return -EBADMSG;
+
+ if (HEADER_CLUSTER_BITS(header) > 21) /* 2MB */
+ return -EBADMSG;
+
+ if (HEADER_SIZE(header) % HEADER_CLUSTER_SIZE(header) != 0)
+ return -EBADMSG;
+
+ if (HEADER_L1_SIZE(header) > 32*1024*1024) /* 32MB */
+ return -EBADMSG;
+
+ if (HEADER_VERSION(header) == 3) {
+
+ if (header->incompatible_features != 0)
+ return -ENOTSUP;
+
+ if (HEADER_HEADER_LENGTH(header) < sizeof(Header))
+ return -EBADMSG;
+ }
+
+ return 0;
+}
+
+int qcow2_convert(int qcow2_fd, int raw_fd) {
+ _cleanup_free_ void *buffer1 = NULL, *buffer2 = NULL;
+ _cleanup_free_ be64_t *l1_table = NULL, *l2_table = NULL;
+ uint64_t sz, i;
+ Header header;
+ ssize_t l;
+ int r;
+
+ l = pread(qcow2_fd, &header, sizeof(header), 0);
+ if (l < 0)
+ return -errno;
+ if (l != sizeof(header))
+ return -EIO;
+
+ r = verify_header(&header);
+ if (r < 0)
+ return r;
+
+ l1_table = new(be64_t, HEADER_L1_SIZE(&header));
+ if (!l1_table)
+ return -ENOMEM;
+
+ l2_table = malloc(HEADER_CLUSTER_SIZE(&header));
+ if (!l2_table)
+ return -ENOMEM;
+
+ buffer1 = malloc(HEADER_CLUSTER_SIZE(&header));
+ if (!buffer1)
+ return -ENOMEM;
+
+ buffer2 = malloc(HEADER_CLUSTER_SIZE(&header));
+ if (!buffer2)
+ return -ENOMEM;
+
+ /* Empty the file if it exists, we rely on zero bits */
+ if (ftruncate(raw_fd, 0) < 0)
+ return -errno;
+
+ if (ftruncate(raw_fd, HEADER_SIZE(&header)) < 0)
+ return -errno;
+
+ sz = sizeof(uint64_t) * HEADER_L1_SIZE(&header);
+ l = pread(qcow2_fd, l1_table, sz, HEADER_L1_TABLE_OFFSET(&header));
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != sz)
+ return -EIO;
+
+ for (i = 0; i < HEADER_L1_SIZE(&header); i ++) {
+ uint64_t l2_begin, j;
+
+ r = normalize_offset(&header, l1_table[i], &l2_begin, NULL, NULL);
+ if (r < 0)
+ return r;
+ if (r == 0)
+ continue;
+
+ l = pread(qcow2_fd, l2_table, HEADER_CLUSTER_SIZE(&header), l2_begin);
+ if (l < 0)
+ return -errno;
+ if ((uint64_t) l != HEADER_CLUSTER_SIZE(&header))
+ return -EIO;
+
+ for (j = 0; j < HEADER_L2_SIZE(&header); j++) {
+ uint64_t data_begin, p, compressed_size;
+ bool compressed;
+
+ p = ((i << HEADER_L2_BITS(&header)) + j) << HEADER_CLUSTER_BITS(&header);
+
+ r = normalize_offset(&header, l2_table[j], &data_begin, &compressed, &compressed_size);
+ if (r < 0)
+ return r;
+ if (r == 0)
+ continue;
+
+ if (compressed)
+ r = decompress_cluster(
+ qcow2_fd, data_begin,
+ raw_fd, p,
+ compressed_size, HEADER_CLUSTER_SIZE(&header),
+ buffer1, buffer2);
+ else
+ r = copy_cluster(
+ qcow2_fd, data_begin,
+ raw_fd, p,
+ HEADER_CLUSTER_SIZE(&header), buffer1);
+ if (r < 0)
+ return r;
+ }
+ }
+
+ return 0;
+}
+
+int qcow2_detect(int fd) {
+ be32_t id;
+ ssize_t l;
+
+ l = pread(fd, &id, sizeof(id), 0);
+ if (l < 0)
+ return -errno;
+ if (l != sizeof(id))
+ return -EIO;
+
+ return htobe32(QCOW2_MAGIC) == id;
+}
diff --git a/src/import/qcow2-util.h b/src/import/qcow2-util.h
new file mode 100644
index 0000000..be7fd1d
--- /dev/null
+++ b/src/import/qcow2-util.h
@@ -0,0 +1,25 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+#pragma once
+
+/***
+ This file is part of systemd.
+
+ Copyright 2015 Lennart Poettering
+
+ systemd is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1 of the License, or
+ (at your option) any later version.
+
+ systemd is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+int qcow2_detect(int fd);
+int qcow2_convert(int qcow2_fd, int raw_fd);
diff --git a/src/import/test-qcow2.c b/src/import/test-qcow2.c
new file mode 100644
index 0000000..9a6c3e8
--- /dev/null
+++ b/src/import/test-qcow2.c
@@ -0,0 +1,55 @@
+/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
+
+/***
+ This file is part of systemd.
+
+ Copyright 2015 Lennart Poettering
+
+ systemd is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1 of the License, or
+ (at your option) any later version.
+
+ systemd is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with systemd; If not, see <http://www.gnu.org/licenses/>.
+***/
+
+#include "log.h"
+#include "util.h"
+
+#include "qcow2-util.h"
+
+int main(int argc, char *argv[]) {
+ _cleanup_close_ int sfd = -1, dfd = -1;
+ int r;
+
+ if (argc != 3) {
+ log_error("Needs two arguments.");
+ return EXIT_FAILURE;
+ }
+
+ sfd = open(argv[1], O_RDONLY|O_CLOEXEC|O_NOCTTY);
+ if (sfd < 0) {
+ log_error_errno(errno, "Can't open source file: %m");
+ return EXIT_FAILURE;
+ }
+
+ dfd = open(argv[2], O_WRONLY|O_CREAT|O_CLOEXEC|O_NOCTTY, 0666);
+ if (dfd < 0) {
+ log_error_errno(errno, "Can't open destination file: %m");
+ return EXIT_FAILURE;
+ }
+
+ r = qcow2_convert(sfd, dfd);
+ if (r < 0) {
+ log_error_errno(r, "Failed to unpack: %m");
+ return EXIT_FAILURE;
+ }
+
+ return EXIT_SUCCESS;
+}
commit 49bb233bb734536b9617d838f09a7bf9b8336003
Author: Lennart Poettering <lennart at poettering.net>
Date: Fri Jan 16 18:42:17 2015 +0100
import: support downloading .xz compressed images
That way we can download fedora cloud raw images as-is and decompress
them on-the-fly.
diff --git a/Makefile.am b/Makefile.am
index 7806a07..ee73598 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5237,13 +5237,15 @@ systemd_import_SOURCES = \
systemd_import_CFLAGS = \
$(AM_CFLAGS) \
- $(LIBCURL_CFLAGS)
+ $(LIBCURL_CFLAGS) \
+ $(XZ_CFLAGS)
systemd_import_LDADD = \
libsystemd-internal.la \
libsystemd-label.la \
libsystemd-shared.la \
- $(LIBCURL_LIBS)
+ $(LIBCURL_LIBS) \
+ $(XZ_LIBS)
endif
endif
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index ac9e6eb..f1b36cb 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -22,6 +22,7 @@
#include <sys/xattr.h>
#include <linux/fs.h>
#include <curl/curl.h>
+#include <lzma.h>
#include "hashmap.h"
#include "utf8.h"
@@ -47,7 +48,11 @@ struct RawImportFile {
char **old_etags;
uint64_t content_length;
- uint64_t written;
+ uint64_t written_compressed;
+ uint64_t written_uncompressed;
+
+ void *payload;
+ size_t payload_size;
usec_t mtime;
@@ -55,6 +60,9 @@ struct RawImportFile {
bool done;
int disk_fd;
+
+ lzma_stream lzma;
+ bool compressed;
};
struct RawImport {
@@ -72,6 +80,8 @@ struct RawImport {
#define FILENAME_ESCAPE "/.#\"\'"
+#define RAW_MAX_SIZE (1024LLU*1024LLU*1024LLU*8) /* 8 GB */
+
static RawImportFile *raw_import_file_unref(RawImportFile *f) {
if (!f)
return NULL;
@@ -93,6 +103,7 @@ static RawImportFile *raw_import_file_unref(RawImportFile *f) {
free(f->local);
free(f->etag);
strv_free(f->old_etags);
+ free(f->payload);
free(f);
return NULL;
@@ -271,7 +282,7 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result
}
if (f->content_length != (uint64_t) -1 &&
- f->content_length != f->written) {
+ f->content_length != f->written_compressed) {
log_error("Download truncated.");
r = -EIO;
goto fail;
@@ -346,50 +357,166 @@ static int raw_import_file_open_disk_for_write(RawImportFile *f) {
return 0;
}
-static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
- RawImportFile *f = userdata;
- size_t sz = size * nmemb;
+static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t sz) {
ssize_t n;
- int r;
- assert(contents);
assert(f);
+ assert(p);
+ assert(sz > 0);
+ assert(f->disk_fd >= 0);
- if (f->done) {
- r = -ESTALE;
- goto fail;
+ if (f->written_uncompressed + sz < f->written_uncompressed) {
+ log_error("File too large, overflow");
+ return -EOVERFLOW;
}
- r = raw_import_file_open_disk_for_write(f);
- if (r < 0)
- goto fail;
+ if (f->written_uncompressed + sz > RAW_MAX_SIZE) {
+ log_error("File overly large, refusing");
+ return -EFBIG;
+ }
+
+ n = write(f->disk_fd, p, sz);
+ if (n < 0) {
+ log_error_errno(errno, "Failed to write file: %m");
+ return -errno;
+ }
+ if ((size_t) n < sz) {
+ log_error("Short write");
+ return -EIO;
+ }
+
+ f->written_uncompressed += sz;
- if (f->written + sz < f->written) {
+ return 0;
+}
+
+static int raw_import_file_write_compressed(RawImportFile *f, void *p, size_t sz) {
+ int r;
+
+ assert(f);
+ assert(p);
+ assert(sz > 0);
+ assert(f->disk_fd >= 0);
+
+ if (f->written_compressed + sz < f->written_compressed) {
log_error("File too large, overflow");
- r = -EOVERFLOW;
- goto fail;
+ return -EOVERFLOW;
}
if (f->content_length != (uint64_t) -1 &&
- f->written + sz > f->content_length) {
+ f->written_compressed + sz > f->content_length) {
log_error("Content length incorrect.");
- r = -EFBIG;
- goto fail;
+ return -EFBIG;
}
- n = write(f->disk_fd, contents, sz);
- if (n < 0) {
- log_error_errno(errno, "Failed to write file: %m");
- goto fail;
+ if (!f->compressed) {
+ r = raw_import_file_write_uncompressed(f, p, sz);
+ if (r < 0)
+ return r;
+ } else {
+ f->lzma.next_in = p;
+ f->lzma.avail_in = sz;
+
+ while (f->lzma.avail_in > 0) {
+ uint8_t buffer[16 * 1024];
+ lzma_ret lzr;
+
+ f->lzma.next_out = buffer;
+ f->lzma.avail_out = sizeof(buffer);
+
+ lzr = lzma_code(&f->lzma, LZMA_RUN);
+ if (lzr != LZMA_OK && lzr != LZMA_STREAM_END) {
+ log_error("Decompression error.");
+ return -EIO;
+ }
+
+ r = raw_import_file_write_uncompressed(f, buffer, sizeof(buffer) - f->lzma.avail_out);
+ if (r < 0)
+ return r;
+ }
}
- if ((size_t) n < sz) {
- log_error("Short write");
- r = -EIO;
+ f->written_compressed += sz;
+
+ return 0;
+}
+
+static int raw_import_file_detect_xz(RawImportFile *f) {
+ static const uint8_t xz_signature[] = {
+ '\xfd', '7', 'z', 'X', 'Z', '\x00'
+ };
+ lzma_ret lzr;
+ int r;
+
+ assert(f);
+
+ if (f->payload_size < sizeof(xz_signature))
+ return 0;
+
+ f->compressed = memcmp(f->payload, xz_signature, sizeof(xz_signature)) == 0;
+ log_debug("Stream is XZ compressed: %s", yes_no(f->compressed));
+
+ if (f->compressed) {
+ lzr = lzma_stream_decoder(&f->lzma, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK);
+ if (lzr != LZMA_OK) {
+ log_error("Failed to initialize LZMA decoder.");
+ return -EIO;
+ }
+ }
+
+ r = raw_import_file_open_disk_for_write(f);
+ if (r < 0)
+ return r;
+
+ r = raw_import_file_write_compressed(f, f->payload, f->payload_size);
+ if (r < 0)
+ return r;
+
+ free(f->payload);
+ f->payload = NULL;
+ f->payload_size = 0;
+
+ return 0;
+}
+
+static size_t raw_import_file_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) {
+ RawImportFile *f = userdata;
+ size_t sz = size * nmemb;
+ int r;
+
+ assert(contents);
+ assert(f);
+
+ if (f->done) {
+ r = -ESTALE;
goto fail;
}
- f->written += sz;
+ if (f->disk_fd < 0) {
+ uint8_t *p;
+
+ /* We haven't opened the file yet, let's first check what it actually is */
+
+ p = realloc(f->payload, f->payload_size + sz);
+ if (!p) {
+ r = log_oom();
+ goto fail;
+ }
+
+ memcpy(p + f->payload_size, contents, sz);
+ f->payload_size = sz;
+ f->payload = p;
+
+ r = raw_import_file_detect_xz(f);
+ if (r < 0)
+ goto fail;
+
+ return sz;
+ }
+
+ r = raw_import_file_write_compressed(f, contents, sz);
+ if (r < 0)
+ goto fail;
return sz;
@@ -438,6 +565,12 @@ static size_t raw_import_file_header_callback(void *contents, size_t size, size_
}
if (r > 0) {
(void) safe_atou64(length, &f->content_length);
+
+ if (f->content_length != (uint64_t) -1) {
+ char bytes[FORMAT_BYTES_MAX];
+ log_info("Downloading %s.", format_bytes(bytes, sizeof(bytes), f->content_length));
+ }
+
return sz;
}
diff --git a/src/import/import.c b/src/import/import.c
index b4d859d..e457adf 100644
--- a/src/import/import.c
+++ b/src/import/import.c
@@ -79,7 +79,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
if (local) {
const char *p;
- suffix = endswith(local, ".raw");
+ suffix = endswith(local, ".raw.xz");
+ if (!suffix)
+ suffix = endswith(local, ".raw");
+ if (!suffix)
+ suffix = endswith(local, ".xz");
if (suffix)
local = strndupa(local, suffix - local);
More information about the systemd-commits
mailing list