[systemd-commits] 10 commits - Makefile.am src/import src/machine src/nspawn src/shared src/test
Lennart Poettering
lennart at kemper.freedesktop.org
Mon Jan 19 11:28:39 PST 2015
Makefile.am | 1
src/import/import-raw.c | 184 +++++++++++++++++++++++++++++---------------
src/import/qcow2-util.c | 6 +
src/machine/image-dbus.c | 4
src/machine/machinectl.c | 20 ++--
src/machine/machined-dbus.c | 2
src/nspawn/nspawn.c | 160 ++++++++++++++++++++++++++++----------
src/shared/btrfs-util.c | 20 ++++
src/shared/btrfs-util.h | 1
src/shared/gpt.h | 2
src/shared/machine-image.c | 8 -
src/shared/machine-image.h | 4
src/shared/util.c | 62 ++++++++++++++
src/shared/util.h | 2
src/test/test-util.c | 37 ++++++++
15 files changed, 391 insertions(+), 122 deletions(-)
New commits:
commit f6c51a8136de3f27e28caea2003e18f4bc4cb9a8
Author: Lennart Poettering <lennart at poettering.net>
Date: Mon Jan 19 20:22:58 2015 +0100
nspawn: support dissecting GPT images that contain only a single generic linux partition
This should allow running Ubuntu UEFI GPT Images with nspawn,
unmodified.
diff --git a/src/nspawn/nspawn.c b/src/nspawn/nspawn.c
index e71d9a8..efeba59 100644
--- a/src/nspawn/nspawn.c
+++ b/src/nspawn/nspawn.c
@@ -2618,7 +2618,8 @@ static int wait_for_block_device(struct udev *udev, dev_t devnum, struct udev_de
#define PARTITION_TABLE_BLURB \
"Note that the disk image needs to either contain only a single MBR partition of\n" \
- "type 0x83 that is marked bootable, or follow\n" \
+ "type 0x83 that is marked bootable, or a sinlge GPT partition of type" \
+ "0FC63DAF-8483-4772-8E79-3D69D8477DE4 or follow\n" \
" http://www.freedesktop.org/wiki/Specifications/DiscoverablePartitionsSpec/\n" \
"to be bootable with systemd-nspawn."
@@ -2637,19 +2638,18 @@ static int dissect_image(
#ifdef GPT_ROOT_SECONDARY
int secondary_root_nr = -1;
#endif
-
- _cleanup_free_ char *home = NULL, *root = NULL, *secondary_root = NULL, *srv = NULL;
+ _cleanup_free_ char *home = NULL, *root = NULL, *secondary_root = NULL, *srv = NULL, *generic = NULL;
_cleanup_udev_enumerate_unref_ struct udev_enumerate *e = NULL;
_cleanup_udev_device_unref_ struct udev_device *d = NULL;
_cleanup_blkid_free_probe_ blkid_probe b = NULL;
_cleanup_udev_unref_ struct udev *udev = NULL;
struct udev_list_entry *first, *item;
- bool home_rw = true, root_rw = true, secondary_root_rw = true, srv_rw = true;
+ bool home_rw = true, root_rw = true, secondary_root_rw = true, srv_rw = true, generic_rw = true;
const char *pttype = NULL;
blkid_partlist pl;
struct stat st;
int r;
- bool is_gpt, is_mbr;
+ bool is_gpt, is_mbr, multiple_generic = false;
assert(fd >= 0);
assert(root_device);
@@ -2769,10 +2769,6 @@ static int dissect_image(
continue;
flags = blkid_partition_get_flags(pp);
- if (is_gpt && (flags & GPT_FLAG_NO_AUTO))
- continue;
- if (is_mbr && (flags != 0x80)) /* Bootable flag */
- continue;
nr = blkid_partition_get_partno(pp);
if (nr < 0)
@@ -2782,6 +2778,9 @@ static int dissect_image(
sd_id128_t type_id;
const char *stype;
+ if (flags & GPT_FLAG_NO_AUTO)
+ continue;
+
stype = blkid_partition_get_type_string(pp);
if (!stype)
continue;
@@ -2841,48 +2840,41 @@ static int dissect_image(
return log_oom();
}
#endif
+ else if (sd_id128_equal(type_id, GPT_LINUX_GENERIC)) {
+
+ if (generic)
+ multiple_generic = true;
+ else {
+ generic_rw = !(flags & GPT_FLAG_READ_ONLY);
+
+ r = free_and_strdup(&generic, node);
+ if (r < 0)
+ return log_oom();
+ }
+ }
} else if (is_mbr) {
int type;
+ if (flags != 0x80) /* Bootable flag */
+ continue;
+
type = blkid_partition_get_type(pp);
if (type != 0x83) /* Linux partition */
continue;
- /* Note that there's a certain, intended
- * asymmetry here: while for GPT we simply
- * take the first valid partition and ignore
- * all others of the same type, for MBR we
- * fail if there are multiple suitable
- * partitions. This is because the GPT
- * partition types are defined by us, and
- * hence we can define their lookup semantics,
- * while for the MBR logic we reuse existing
- * definitions, and simply don't want to make
- * out the situation. */
-
- if (root) {
- log_error("Identified multiple bootable Linux 0x83 partitions on\n"
- " %s\n"
- PARTITION_TABLE_BLURB, arg_image);
- return -EINVAL;
- }
-
- root_nr = nr;
+ if (generic)
+ multiple_generic = true;
+ else {
+ generic_rw = true;
- r = free_and_strdup(&root, node);
- if (r < 0)
- return log_oom();
+ r = free_and_strdup(&root, node);
+ if (r < 0)
+ return log_oom();
+ }
}
}
- if (!root && !secondary_root) {
- log_error("Failed to identify root partition in disk image\n"
- " %s\n"
- PARTITION_TABLE_BLURB, arg_image);
- return -EINVAL;
- }
-
if (root) {
*root_device = root;
root = NULL;
@@ -2895,6 +2887,31 @@ static int dissect_image(
*root_device_rw = secondary_root_rw;
*secondary = true;
+ } else if (generic) {
+
+ /* There were no partitions with precise meanings
+ * around, but we found generic partitions. In this
+ * case, if there's only one, we can go ahead and boot
+ * it, otherwise we bail out, because we really cannot
+ * make any sense of it. */
+
+ if (multiple_generic) {
+ log_error("Identified multiple bootable Linux partitions on\n"
+ " %s\n"
+ PARTITION_TABLE_BLURB, arg_image);
+ return -EINVAL;
+ }
+
+ *root_device = generic;
+ generic = NULL;
+
+ *root_device_rw = generic_rw;
+ *secondary = false;
+ } else {
+ log_error("Failed to identify root partition in disk image\n"
+ " %s\n"
+ PARTITION_TABLE_BLURB, arg_image);
+ return -EINVAL;
}
if (home) {
diff --git a/src/shared/gpt.h b/src/shared/gpt.h
index ef3444f..87308b0 100644
--- a/src/shared/gpt.h
+++ b/src/shared/gpt.h
@@ -57,3 +57,5 @@
* just because we saw no point in defining any other values here. */
#define GPT_FLAG_READ_ONLY (1ULL << 60)
#define GPT_FLAG_NO_AUTO (1ULL << 63)
+
+#define GPT_LINUX_GENERIC SD_ID128_MAKE(0f,c6,3d,af,84,83,47,72,8e,79,3d,69,d8,47,7d,e4)
commit ec5cb56ee18f529a023deedd0806854f08499b6a
Author: Lennart Poettering <lennart at poettering.net>
Date: Mon Jan 19 20:22:50 2015 +0100
import: clarify when we are unpacking the qcow2 device
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index da72725..80fdbb7 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -264,6 +264,8 @@ static int raw_import_maybe_convert_qcow2(RawImportFile *f) {
if (converted_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", t);
+ log_info("Unpacking QCOW2 file.");
+
r = qcow2_convert(f->disk_fd, converted_fd);
if (r < 0) {
unlink(t);
commit 2fbe4296c5ba5bcce3ac845f196c60a88e3181fe
Author: Lennart Poettering <lennart at poettering.net>
Date: Mon Jan 19 20:21:57 2015 +0100
inspawn: wait until udev has probed a loopback device before making us of it
diff --git a/src/nspawn/nspawn.c b/src/nspawn/nspawn.c
index e2ce9fb..e71d9a8 100644
--- a/src/nspawn/nspawn.c
+++ b/src/nspawn/nspawn.c
@@ -43,6 +43,8 @@
#include <linux/veth.h>
#include <sys/personality.h>
#include <linux/loop.h>
+#include <poll.h>
+#include <sys/file.h>
#ifdef HAVE_SELINUX
#include <selinux/selinux.h>
@@ -2557,6 +2559,63 @@ static int setup_image(char **device_path, int *loop_nr) {
return r;
}
+static int wait_for_block_device(struct udev *udev, dev_t devnum, struct udev_device **ret) {
+ _cleanup_udev_monitor_unref_ struct udev_monitor *monitor = NULL;
+ int r;
+
+ assert(udev);
+ assert(ret);
+
+ for (;;) {
+ _cleanup_udev_device_unref_ struct udev_device *d = NULL;
+ struct pollfd pfd = {
+ .events = POLLIN
+ };
+
+ d = udev_device_new_from_devnum(udev, 'b', devnum);
+ if (!d)
+ return log_oom();
+
+ r = udev_device_get_is_initialized(d);
+ if (r < 0)
+ return log_error_errno(r, "Failed to check if device is initialized: %m");
+ if (r > 0) {
+ *ret = d;
+ d = NULL;
+ return 0;
+ }
+ d = udev_device_unref(d);
+
+ if (!monitor) {
+ monitor = udev_monitor_new_from_netlink(udev, "udev");
+ if (!monitor)
+ return log_oom();
+
+ r = udev_monitor_filter_add_match_subsystem_devtype(monitor, "block", NULL);
+ if (r < 0)
+ return log_error_errno(r, "Failed to add block match: %m");
+
+ r = udev_monitor_enable_receiving(monitor);
+ if (r < 0)
+ return log_error_errno(r, "Failed to turn on monitor: %m");
+
+ continue;
+ }
+
+ pfd.fd = udev_monitor_get_fd(monitor);
+ if (pfd.fd < 0)
+ return log_error_errno(r, "Failed to get udev monitor fd: %m");
+
+ r = poll(&pfd, 1, -1);
+ if (r < 0)
+ return log_error_errno(errno, "Failed to wait for device initialization: %m");
+
+ d = udev_monitor_receive_device(monitor);
+ }
+
+ return 0;
+}
+
#define PARTITION_TABLE_BLURB \
"Note that the disk image needs to either contain only a single MBR partition of\n" \
"type 0x83 that is marked bootable, or follow\n" \
@@ -2659,9 +2718,9 @@ static int dissect_image(
if (fstat(fd, &st) < 0)
return log_error_errno(errno, "Failed to stat block device: %m");
- d = udev_device_new_from_devnum(udev, 'b', st.st_rdev);
- if (!d)
- return log_oom();
+ r = wait_for_block_device(udev, st.st_rdev, &d);
+ if (r < 0)
+ return r;
e = udev_enumerate_new(udev);
if (!e)
commit 0716faad4a846d5f3cdce4bf37648d3254b9332f
Author: Lennart Poettering <lennart at poettering.net>
Date: Mon Jan 19 17:02:37 2015 +0100
import: make sure don't leak the LZMA context
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index 361e30a..da72725 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -104,6 +104,7 @@ static RawImportFile *raw_import_file_unref(RawImportFile *f) {
free(f->temp_path);
}
+ lzma_end(&f->lzma);
free(f->url);
free(f->local);
free(f->etag);
commit c19de71113f956809995fc68817e055e9f61f607
Author: Lennart Poettering <lennart at poettering.net>
Date: Mon Jan 19 17:01:15 2015 +0100
machined: refer to the disk space allocated for an image to "usage" rather than "size"
After all, it's closer to the "du"-reported value than to the file
sizes...
diff --git a/src/machine/image-dbus.c b/src/machine/image-dbus.c
index 659f7de..f5c7d4d 100644
--- a/src/machine/image-dbus.c
+++ b/src/machine/image-dbus.c
@@ -135,9 +135,9 @@ const sd_bus_vtable image_vtable[] = {
SD_BUS_PROPERTY("ReadOnly", "b", bus_property_get_bool, offsetof(Image, read_only), 0),
SD_BUS_PROPERTY("CreationTimestamp", "t", NULL, offsetof(Image, crtime), 0),
SD_BUS_PROPERTY("ModificationTimestamp", "t", NULL, offsetof(Image, mtime), 0),
- SD_BUS_PROPERTY("Size", "t", NULL, offsetof(Image, size), 0),
+ SD_BUS_PROPERTY("Usage", "t", NULL, offsetof(Image, usage), 0),
SD_BUS_PROPERTY("Limit", "t", NULL, offsetof(Image, limit), 0),
- SD_BUS_PROPERTY("SizeExclusive", "t", NULL, offsetof(Image, size_exclusive), 0),
+ SD_BUS_PROPERTY("UsageExclusive", "t", NULL, offsetof(Image, usage_exclusive), 0),
SD_BUS_PROPERTY("LimitExclusive", "t", NULL, offsetof(Image, limit_exclusive), 0),
SD_BUS_METHOD("Remove", NULL, NULL, bus_image_method_remove, 0),
SD_BUS_METHOD("Rename", "s", NULL, bus_image_method_rename, 0),
diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c
index 7ea991a..bae2abe 100644
--- a/src/machine/machinectl.c
+++ b/src/machine/machinectl.c
@@ -214,7 +214,7 @@ static int compare_image_info(const void *a, const void *b) {
static int list_images(int argc, char *argv[], void *userdata) {
_cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL;
- size_t max_name = strlen("NAME"), max_type = strlen("TYPE"), max_size = strlen("SIZE"), max_crtime = strlen("CREATED"), max_mtime = strlen("MODIFIED");
+ size_t max_name = strlen("NAME"), max_type = strlen("TYPE"), max_size = strlen("USAGE"), max_crtime = strlen("CREATED"), max_mtime = strlen("MODIFIED");
_cleanup_bus_message_unref_ sd_bus_message *reply = NULL;
_cleanup_free_ ImageInfo *images = NULL;
size_t n_images = 0, n_allocated = 0, j;
@@ -304,7 +304,7 @@ static int list_images(int argc, char *argv[], void *userdata) {
(int) max_name, "NAME",
(int) max_type, "TYPE",
"RO",
- (int) max_size, "SIZE",
+ (int) max_size, "USAGE",
(int) max_crtime, "CREATED",
(int) max_mtime, "MODIFIED");
@@ -741,9 +741,9 @@ typedef struct ImageStatusInfo {
int read_only;
usec_t crtime;
usec_t mtime;
- uint64_t size;
+ uint64_t usage;
uint64_t limit;
- uint64_t size_exclusive;
+ uint64_t usage_exclusive;
uint64_t limit_exclusive;
} ImageStatusInfo;
@@ -786,12 +786,12 @@ static void print_image_status_info(sd_bus *bus, ImageStatusInfo *i) {
else if (s2)
printf("\tModified: %s\n", s2);
- s3 = format_bytes(bs, sizeof(bs), i->size);
- s4 = i->size_exclusive != i->size ? format_bytes(bs_exclusive, sizeof(bs_exclusive), i->size_exclusive) : NULL;
+ s3 = format_bytes(bs, sizeof(bs), i->usage);
+ s4 = i->usage_exclusive != i->usage ? format_bytes(bs_exclusive, sizeof(bs_exclusive), i->usage_exclusive) : NULL;
if (s3 && s4)
- printf("\t Size: %s (exclusive: %s)\n", s3, s4);
+ printf("\t Usage: %s (exclusive: %s)\n", s3, s4);
else if (s3)
- printf("\t Size: %s\n", s3);
+ printf("\t Usage: %s\n", s3);
s3 = format_bytes(bs, sizeof(bs), i->limit);
s4 = i->limit_exclusive != i->limit ? format_bytes(bs_exclusive, sizeof(bs_exclusive), i->limit_exclusive) : NULL;
@@ -810,9 +810,9 @@ static int show_image_info(const char *verb, sd_bus *bus, const char *path, bool
{ "ReadOnly", "b", NULL, offsetof(ImageStatusInfo, read_only) },
{ "CreationTimestamp", "t", NULL, offsetof(ImageStatusInfo, crtime) },
{ "ModificationTimestamp", "t", NULL, offsetof(ImageStatusInfo, mtime) },
- { "Size", "t", NULL, offsetof(ImageStatusInfo, size) },
+ { "Usage", "t", NULL, offsetof(ImageStatusInfo, usage) },
{ "Limit", "t", NULL, offsetof(ImageStatusInfo, limit) },
- { "SizeExclusive", "t", NULL, offsetof(ImageStatusInfo, size_exclusive) },
+ { "UsageExclusive", "t", NULL, offsetof(ImageStatusInfo, usage_exclusive) },
{ "LimitExclusive", "t", NULL, offsetof(ImageStatusInfo, limit_exclusive) },
{}
};
diff --git a/src/machine/machined-dbus.c b/src/machine/machined-dbus.c
index 233c6e8..ac19695 100644
--- a/src/machine/machined-dbus.c
+++ b/src/machine/machined-dbus.c
@@ -504,7 +504,7 @@ static int method_list_images(sd_bus *bus, sd_bus_message *message, void *userda
image->read_only,
image->crtime,
image->mtime,
- image->size,
+ image->usage,
p);
if (r < 0)
return r;
diff --git a/src/shared/machine-image.c b/src/shared/machine-image.c
index 5112d24..8ea6105 100644
--- a/src/shared/machine-image.c
+++ b/src/shared/machine-image.c
@@ -73,7 +73,7 @@ static int image_new(
i->read_only = read_only;
i->crtime = crtime;
i->mtime = mtime;
- i->size = i->size_exclusive = (uint64_t) -1;
+ i->usage = i->usage_exclusive = (uint64_t) -1;
i->limit = i->limit_exclusive = (uint64_t) -1;
i->name = strdup(pretty);
@@ -164,8 +164,8 @@ static int image_make(
r = btrfs_subvol_get_quota_fd(fd, "a);
if (r >= 0) {
- (*ret)->size = quota.referred;
- (*ret)->size_exclusive = quota.exclusive;
+ (*ret)->usage = quota.referred;
+ (*ret)->usage_exclusive = quota.exclusive;
(*ret)->limit = quota.referred_max;
(*ret)->limit_exclusive = quota.exclusive_max;
@@ -218,7 +218,7 @@ static int image_make(
if (r < 0)
return r;
- (*ret)->size = (*ret)->size_exclusive = st.st_blocks * 512;
+ (*ret)->usage = (*ret)->usage_exclusive = st.st_blocks * 512;
(*ret)->limit = (*ret)->limit_exclusive = st.st_size;
return 1;
diff --git a/src/shared/machine-image.h b/src/shared/machine-image.h
index c15b034..75fa5f4 100644
--- a/src/shared/machine-image.h
+++ b/src/shared/machine-image.h
@@ -41,8 +41,8 @@ typedef struct Image {
usec_t crtime;
usec_t mtime;
- uint64_t size;
- uint64_t size_exclusive;
+ uint64_t usage;
+ uint64_t usage_exclusive;
uint64_t limit;
uint64_t limit_exclusive;
} Image;
commit 1c7dd82563ff2e71a067aea20d2acb2d0553644b
Author: Lennart Poettering <lennart at poettering.net>
Date: Sat Jan 17 18:11:45 2015 +0100
qcow2: when dissecting qcow2, use btrfs clone ioctls for reflinking blocks to target
diff --git a/Makefile.am b/Makefile.am
index ce5ebf7..37ea845 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -5285,6 +5285,7 @@ test_qcow2_CFLAGS = \
test_qcow2_LDADD = \
libsystemd-internal.la \
+ libsystemd-label.la \
libsystemd-shared.la \
$(ZLIB_LIBS)
endif
diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c
index c84c6aa..9b0c23b 100644
--- a/src/import/qcow2-util.c
+++ b/src/import/qcow2-util.c
@@ -24,6 +24,7 @@
#include "util.h"
#include "sparse-endian.h"
#include "qcow2-util.h"
+#include "btrfs-util.h"
#define QCOW2_MAGIC 0x514649fb
@@ -85,6 +86,11 @@ static int copy_cluster(
void *buffer) {
ssize_t l;
+ int r;
+
+ r = btrfs_clone_range(sfd, soffset, dfd, doffset, cluster_size);
+ if (r >= 0)
+ return r;
l = pread(sfd, buffer, cluster_size, soffset);
if (l < 0)
diff --git a/src/shared/btrfs-util.c b/src/shared/btrfs-util.c
index bd100ee..254483c 100644
--- a/src/shared/btrfs-util.c
+++ b/src/shared/btrfs-util.c
@@ -275,6 +275,26 @@ int btrfs_reflink(int infd, int outfd) {
return 0;
}
+int btrfs_clone_range(int infd, uint64_t in_offset, int outfd, uint64_t out_offset, uint64_t sz) {
+ struct btrfs_ioctl_clone_range_args args = {
+ .src_fd = infd,
+ .src_offset = in_offset,
+ .src_length = sz,
+ .dest_offset = out_offset,
+ };
+ int r;
+
+ assert(infd >= 0);
+ assert(outfd >= 0);
+ assert(sz > 0);
+
+ r = ioctl(outfd, BTRFS_IOC_CLONE_RANGE, &args);
+ if (r < 0)
+ return -errno;
+
+ return 0;
+}
+
int btrfs_get_block_device(const char *path, dev_t *dev) {
struct btrfs_ioctl_fs_info_args fsi = {};
_cleanup_close_ int fd = -1;
diff --git a/src/shared/btrfs-util.h b/src/shared/btrfs-util.h
index 1bff917..28946c6 100644
--- a/src/shared/btrfs-util.h
+++ b/src/shared/btrfs-util.h
@@ -55,6 +55,7 @@ int btrfs_subvol_get_info_fd(int fd, BtrfsSubvolInfo *info);
int btrfs_subvol_get_quota_fd(int fd, BtrfsQuotaInfo *quota);
int btrfs_reflink(int infd, int outfd);
+int btrfs_clone_range(int infd, uint64_t in_offset, int ofd, uint64_t out_offset, uint64_t sz);
int btrfs_get_block_device(const char *path, dev_t *dev);
commit 1e20b41187ff7d27477b5322690e447753c66ace
Author: Lennart Poettering <lennart at poettering.net>
Date: Sat Jan 17 03:22:06 2015 +0100
import: when downloading raw files, show simple progress reports
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index 94b936b..361e30a 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -64,6 +64,10 @@ struct RawImportFile {
lzma_stream lzma;
bool compressed;
+
+ unsigned progress_percent;
+ usec_t start_usec;
+ usec_t last_status_usec;
};
struct RawImport {
@@ -650,6 +654,40 @@ fail:
return 0;
}
+static int raw_import_file_progress_callback(void *userdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) {
+ RawImportFile *f = userdata;
+ unsigned percent;
+ usec_t n;
+
+ assert(f);
+
+ if (dltotal <= 0)
+ return 0;
+
+ percent = ((100 * dlnow) / dltotal);
+ n = now(CLOCK_MONOTONIC);
+
+ if (n > f->last_status_usec + USEC_PER_SEC &&
+ percent != f->progress_percent) {
+ char buf[FORMAT_TIMESPAN_MAX];
+
+ if (n - f->start_usec > USEC_PER_SEC && dlnow > 0) {
+ usec_t left, done;
+
+ done = n - f->start_usec;
+ left = (usec_t) (((double) done * (double) dltotal) / dlnow) - done;
+
+ log_info("Got %u%%. %s left.", percent, format_timespan(buf, sizeof(buf), left, USEC_PER_SEC));
+ } else
+ log_info("Got %u%%.", percent);
+
+ f->progress_percent = percent;
+ f->last_status_usec = n;
+ }
+
+ return 0;
+}
+
static bool etag_is_valid(const char *etag) {
if (!endswith(etag, "\""))
@@ -770,6 +808,15 @@ static int raw_import_file_begin(RawImportFile *f) {
if (curl_easy_setopt(f->curl, CURLOPT_HEADERDATA, f) != CURLE_OK)
return -EIO;
+ if (curl_easy_setopt(f->curl, CURLOPT_XFERINFOFUNCTION, raw_import_file_progress_callback) != CURLE_OK)
+ return -EIO;
+
+ if (curl_easy_setopt(f->curl, CURLOPT_XFERINFODATA, f) != CURLE_OK)
+ return -EIO;
+
+ if (curl_easy_setopt(f->curl, CURLOPT_NOPROGRESS, 0) != CURLE_OK)
+ return -EIO;
+
r = curl_glue_add(f->import->glue, f->curl);
if (r < 0)
return r;
@@ -871,6 +918,7 @@ int raw_import_pull(RawImport *import, const char *url, const char *local, bool
f->import = import;
f->disk_fd = -1;
f->content_length = (uint64_t) -1;
+ f->start_usec = now(CLOCK_MONOTONIC);
f->url = strdup(url);
if (!f->url)
commit 2f64ba0e6ea4fe8bac69215290d16321ab820f54
Author: Lennart Poettering <lennart at poettering.net>
Date: Sat Jan 17 02:45:42 2015 +0100
import: simplify the code a bit
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index c82d263..94b936b 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -152,82 +152,81 @@ static int raw_import_file_make_final_path(RawImportFile *f) {
return 0;
}
-static void raw_import_file_success(RawImportFile *f) {
+static int raw_import_file_make_local_copy(RawImportFile *f) {
+ _cleanup_free_ char *tp = NULL;
+ _cleanup_close_ int dfd = -1;
+ const char *p;
int r;
assert(f);
- f->done = true;
+ if (!f->local)
+ return 0;
- if (f->local) {
- _cleanup_free_ char *tp = NULL;
- _cleanup_close_ int dfd = -1;
- const char *p;
+ if (f->disk_fd >= 0) {
+ if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1)
+ return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
+ } else {
+ r = raw_import_file_make_final_path(f);
+ if (r < 0)
+ return log_oom();
- if (f->disk_fd >= 0) {
- if (lseek(f->disk_fd, SEEK_SET, 0) == (off_t) -1) {
- r = log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
- goto finish;
- }
- } else {
- r = raw_import_file_make_final_path(f);
- if (r < 0) {
- log_oom();
- goto finish;
- }
+ f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
+ if (f->disk_fd < 0)
+ return log_error_errno(errno, "Failed to open vendor image: %m");
+ }
- f->disk_fd = open(f->final_path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
- if (f->disk_fd < 0) {
- r = log_error_errno(errno, "Failed to open vendor image: %m");
- goto finish;
- }
- }
+ p = strappenda(f->import->image_root, "/", f->local, ".raw");
+ if (f->force_local)
+ (void) rm_rf_dangerous(p, false, true, false);
- p = strappenda(f->import->image_root, "/", f->local, ".raw");
- if (f->force_local)
- (void) rm_rf_dangerous(p, false, true, false);
+ r = tempfn_random(p, &tp);
+ if (r < 0)
+ return log_oom();
- r = tempfn_random(p, &tp);
- if (r < 0) {
- log_oom();
- goto finish;
- }
+ dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
+ if (dfd < 0)
+ return log_error_errno(errno, "Failed to create writable copy of image: %m");
- dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
- if (dfd < 0) {
- r = log_error_errno(errno, "Failed to create writable copy of image: %m");
- goto finish;
- }
+ /* Turn off COW writing. This should greatly improve
+ * performance on COW file systems like btrfs, since it
+ * reduces fragmentation caused by not allowing in-place
+ * writes. */
+ r = chattr_fd(dfd, true, FS_NOCOW_FL);
+ if (r < 0)
+ log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
- /* Turn off COW writing. This should greatly improve
- * performance on COW file systems like btrfs, since it
- * reduces fragmentation caused by not allowing in-place
- * writes. */
- r = chattr_fd(dfd, true, FS_NOCOW_FL);
- if (r < 0)
- log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
+ r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
+ if (r < 0) {
+ unlink(tp);
+ return log_error_errno(r, "Failed to make writable copy of image: %m");
+ }
- r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
- if (r < 0) {
- log_error_errno(r, "Failed to make writable copy of image: %m");
- unlink(tp);
- goto finish;
- }
+ (void) copy_times(f->disk_fd, dfd);
+ (void) copy_xattr(f->disk_fd, dfd);
- (void) copy_times(f->disk_fd, dfd);
- (void) copy_xattr(f->disk_fd, dfd);
+ dfd = safe_close(dfd);
- dfd = safe_close(dfd);
+ r = rename(tp, p);
+ if (r < 0) {
+ unlink(tp);
+ return log_error_errno(errno, "Failed to move writable image into place: %m");
+ }
- r = rename(tp, p);
- if (r < 0) {
- r = log_error_errno(errno, "Failed to move writable image into place: %m");
- unlink(tp);
- goto finish;
- }
+ log_info("Created new local image %s.", p);
+ return 0;
+}
- log_info("Created new local image %s.", p);
- }
+static void raw_import_file_success(RawImportFile *f) {
+ int r;
+
+ assert(f);
+
+ f->done = true;
+
+ r = raw_import_file_make_local_copy(f);
+ if (r < 0)
+ goto finish;
f->disk_fd = safe_close(f->disk_fd);
r = 0;
commit ff6a74609b7c925834da1373d3adb9642ca51422
Author: Lennart Poettering <lennart at poettering.net>
Date: Sat Jan 17 02:36:23 2015 +0100
import-raw: when downloading raw images, generate sparse files if we can
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index f830ba4..c82d263 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -332,6 +332,14 @@ static void raw_import_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result
goto fail;
}
+ /* Make sure the file size is right, in case the file was
+ * sparse and we just seeked for the last part */
+ if (ftruncate(f->disk_fd, f->written_uncompressed) < 0) {
+ log_error_errno(errno, "Failed to truncate file: %m");
+ r = -errno;
+ goto fail;
+ }
+
r = raw_import_maybe_convert_qcow2(f);
if (r < 0)
goto fail;
@@ -427,7 +435,7 @@ static int raw_import_file_write_uncompressed(RawImportFile *f, void *p, size_t
return -EFBIG;
}
- n = write(f->disk_fd, p, sz);
+ n = sparse_write(f->disk_fd, p, sz, 64);
if (n < 0) {
log_error_errno(errno, "Failed to write file: %m");
return -errno;
diff --git a/src/shared/util.c b/src/shared/util.c
index 8f6d5e6..fd54023 100644
--- a/src/shared/util.c
+++ b/src/shared/util.c
@@ -7931,3 +7931,65 @@ void release_lock_file(LockFile *f) {
f->fd = safe_close(f->fd);
f->operation = 0;
}
+
+static size_t nul_length(const uint8_t *p, size_t sz) {
+ size_t n = 0;
+
+ while (sz > 0) {
+ if (*p != 0)
+ break;
+
+ n++;
+ p++;
+ sz--;
+ }
+
+ return n;
+}
+
+ssize_t sparse_write(int fd, const void *p, size_t sz, size_t run_length) {
+ const uint8_t *q, *w, *e;
+ ssize_t l;
+
+ q = w = p;
+ e = q + sz;
+ while (q < e) {
+ size_t n;
+
+ n = nul_length(q, e - q);
+
+ /* If there are more than the specified run length of
+ * NUL bytes, or if this is the beginning or the end
+ * of the buffer, then seek instead of write */
+ if ((n > run_length) ||
+ (n > 0 && q == p) ||
+ (n > 0 && q + n >= e)) {
+ if (q > w) {
+ l = write(fd, w, q - w);
+ if (l < 0)
+ return -errno;
+ if (l != q -w)
+ return -EIO;
+ }
+
+ if (lseek(fd, n, SEEK_CUR) == (off_t) -1)
+ return -errno;
+
+ q += n;
+ w = q;
+ } else if (n > 0)
+ q += n;
+ else
+ q ++;
+ }
+
+ if (q > w) {
+ l = write(fd, w, q - w);
+ if (l < 0)
+ return -errno;
+ if (l != q - w)
+ return -EIO;
+ }
+
+ return q - (const uint8_t*) p;
+}
diff --git a/src/shared/util.h b/src/shared/util.h
index 8a3e95a..2e662c9 100644
--- a/src/shared/util.h
+++ b/src/shared/util.h
@@ -1062,3 +1062,5 @@ void release_lock_file(LockFile *f);
#define LOCK_FILE_INIT { .fd = -1, .path = NULL }
#define RLIMIT_MAKE_CONST(lim) ((struct rlimit) { lim, lim })
+
+ssize_t sparse_write(int fd, const void *p, size_t sz, size_t run_length);
diff --git a/src/test/test-util.c b/src/test/test-util.c
index 4bb5154..0c0d2f6 100644
--- a/src/test/test-util.c
+++ b/src/test/test-util.c
@@ -1464,6 +1464,42 @@ static void test_uid_ptr(void) {
assert_se(PTR_TO_UID(UID_TO_PTR(1000)) == 1000);
}
+static void test_sparse_write_one(int fd, const char *buffer, size_t n) {
+ char check[n];
+
+ assert_se(lseek(fd, 0, SEEK_SET) == 0);
+ assert_se(ftruncate(fd, 0) >= 0);
+ assert_se(sparse_write(fd, buffer, n, 4) == (ssize_t) n);
+
+ assert_se(lseek(fd, 0, SEEK_CUR) == (off_t) n);
+ assert_se(ftruncate(fd, n) >= 0);
+
+ assert_se(lseek(fd, 0, SEEK_SET) == 0);
+ assert_se(read(fd, check, n) == (ssize_t) n);
+
+ assert_se(memcmp(buffer, check, n) == 0);
+}
+
+static void test_sparse_write(void) {
+ const char test_a[] = "test";
+ const char test_b[] = "\0\0\0\0test\0\0\0\0";
+ const char test_c[] = "\0\0test\0\0\0\0";
+ const char test_d[] = "\0\0test\0\0\0test\0\0\0\0test\0\0\0\0\0test\0\0\0test\0\0\0\0test\0\0\0\0\0\0\0\0";
+ const char test_e[] = "test\0\0\0\0test";
+ _cleanup_close_ int fd = -1;
+ char fn[] = "/tmp/sparseXXXXXX";
+
+ fd = mkostemp(fn, O_CLOEXEC);
+ assert_se(fd >= 0);
+ unlink(fn);
+
+ test_sparse_write_one(fd, test_a, sizeof(test_a));
+ test_sparse_write_one(fd, test_b, sizeof(test_b));
+ test_sparse_write_one(fd, test_c, sizeof(test_c));
+ test_sparse_write_one(fd, test_d, sizeof(test_d));
+ test_sparse_write_one(fd, test_e, sizeof(test_e));
+}
+
int main(int argc, char *argv[]) {
log_parse_environment();
log_open();
@@ -1540,6 +1576,7 @@ int main(int argc, char *argv[]) {
test_raw_clone();
test_same_fd();
test_uid_ptr();
+ test_sparse_write();
return 0;
}
commit 47bc4fd86d1119945be3d0217f3795c901b7ffff
Author: Lennart Poettering <lennart at poettering.net>
Date: Sat Jan 17 02:34:33 2015 +0100
import-raw: set NOCOW flag on all raw images we create
diff --git a/src/import/import-raw.c b/src/import/import-raw.c
index c15765d..f830ba4 100644
--- a/src/import/import-raw.c
+++ b/src/import/import-raw.c
@@ -205,7 +205,7 @@ static void raw_import_file_success(RawImportFile *f) {
* writes. */
r = chattr_fd(dfd, true, FS_NOCOW_FL);
if (r < 0)
- log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
+ log_warning_errno(errno, "Failed to set file attributes on %s: %m", tp);
r = copy_bytes(f->disk_fd, dfd, (off_t) -1, true);
if (r < 0) {
@@ -402,6 +402,10 @@ static int raw_import_file_open_disk_for_write(RawImportFile *f) {
if (f->disk_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", f->temp_path);
+ r = chattr_fd(f->disk_fd, true, FS_NOCOW_FL);
+ if (r < 0)
+ log_warning_errno(errno, "Failed to set file attributes on %s: %m", f->temp_path);
+
return 0;
}
More information about the systemd-commits
mailing list