shaba/systemd
1
0
Fork 0
mirror of https://github.com/systemd/systemd.git synced 2024-12-25 01:34:28 +03:00
Code

import: add new "--direct" mode + add controls for turning certain features on/off

Browse Source 
This reworks/modernizes the tar/raw import logic and adds the following
new features:

- Adds the ability to control btrfs subvol and quota behaviour which was
  previously always on via an env var and cmdline arg

- Adds control whether to sync() stuff after writing it, similar via env
  var + cmdline arg

- Similar, the QCOW2 unpacking logic that was previously the implied
  default may now be controlled via env var + cmdline arg.

- adds a "direct" mode. In this mode, the systemd-import tool can be
  used as a simple tool for decompressing/unpacking/installing arbitrary
  files, without all the additional meta data and auxiliary resources,
  i.e.  outside of the immediate disk image context. Via the new
  --offset= and --size-max= switches the downloaded data can be written
  to specific locations of a file (which is particularly useful to use
  the tool to download fs images and write them to a partition location
  before actually creating the partition).

We'll later use the latter feature for "sysupdate" concept, where images
can be directly be written to partitions. That way the systemd-import
binary will be used as backend for both "systemd-importd" and
"systemd-sysupdate" and share most of the same code.
This commit is contained in:
Lennart Poettering 2021-01-22 17:40:51 +01:00
parent 235be6bcea
commit d32a5841fb
6 changed files with 548 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/import/import-common.c
View File

@ -13,6 +13,7 @@
#include "fd-util.h" #include "fd-util.h"
#include "fileio.h" #include "fileio.h"
#include "fs-util.h" #include "fs-util.h"
#include "hostname-util.h"
#include "import-common.h" #include "import-common.h"
#include "os-util.h" #include "os-util.h"
#include "process-util.h" #include "process-util.h"
@ -327,3 +328,38 @@ int import_mangle_os_tree(const char *path) {
return 0; return 0;
} }
bool import_validate_local(const char *name, ImportFlags flags) {
/* By default we insist on a valid hostname for naming images. But optionally we relax that, in which
* case it can be any path name */
if (FLAGS_SET(flags, IMPORT_DIRECT))
return path_is_valid(name);
return hostname_is_valid(name, 0);
}
static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
log_notice("Transfer aborted.");
sd_event_exit(sd_event_source_get_event(s), EINTR);
return 0;
}
int import_allocate_event_with_signals(sd_event **ret) {
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
int r;
assert(ret);
r = sd_event_default(&event);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
*ret = TAKE_PTR(event);
return 0;
}

20
src/import/import-common.h
View File

@ -3,11 +3,19 @@
#include <sys/types.h> #include <sys/types.h>
typedef enum ImportFlags { #include "sd-event.h"
IMPORT_FORCE = 1 << 0, /* replace existing image */
IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */
IMPORT_FLAGS_MASK = IMPORT_FORCE|IMPORT_READ_ONLY, typedef enum ImportFlags {
IMPORT_FORCE = 1 << 0, /* replace existing image */
IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */
IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */
IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
IMPORT_DIRECT = 1 << 5, /* import without rename games */
IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */
IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC,
IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC,
} ImportFlags; } ImportFlags;
int import_make_read_only_fd(int fd); int import_make_read_only_fd(int fd);
@ -17,3 +25,7 @@ int import_fork_tar_c(const char *path, pid_t *ret);
int import_fork_tar_x(const char *path, pid_t *ret); int import_fork_tar_x(const char *path, pid_t *ret);
int import_mangle_os_tree(const char *path); int import_mangle_os_tree(const char *path);
bool import_validate_local(const char *name, ImportFlags flags);
int import_allocate_event_with_signals(sd_event **ret);

217
src/import/import-raw.c
View File

@ -14,6 +14,7 @@
#include "import-common.h" #include "import-common.h"
#include "import-compress.h" #include "import-compress.h"
#include "import-raw.h" #include "import-raw.h"
#include "install-file.h"
#include "io-util.h" #include "io-util.h"
#include "machine-pool.h" #include "machine-pool.h"
#include "mkdir.h" #include "mkdir.h"
@ -52,23 +53,27 @@ struct RawImport {
uint64_t written_compressed; uint64_t written_compressed;
uint64_t written_uncompressed; uint64_t written_uncompressed;
struct stat st; struct stat input_stat;
struct stat output_stat;
unsigned last_percent; unsigned last_percent;
RateLimit progress_ratelimit; RateLimit progress_ratelimit;
uint64_t offset;
uint64_t size_max;
}; };
RawImport* raw_import_unref(RawImport *i) { RawImport* raw_import_unref(RawImport *i) {
if (!i) if (!i)
return NULL; return NULL;
sd_event_unref(i->event); sd_event_source_unref(i->input_event_source);
unlink_and_free(i->temp_path); unlink_and_free(i->temp_path);
import_compress_free(&i->compress); import_compress_free(&i->compress);
sd_event_source_unref(i->input_event_source); sd_event_unref(i->event);
safe_close(i->output_fd); safe_close(i->output_fd);
@ -107,6 +112,8 @@ int raw_import_new(
.last_percent = UINT_MAX, .last_percent = UINT_MAX,
.image_root = TAKE_PTR(root), .image_root = TAKE_PTR(root),
.progress_ratelimit = { 100 * USEC_PER_MSEC, 1 }, .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 },
.offset = UINT64_MAX,
.size_max = UINT64_MAX,
}; };
if (event) if (event)
@ -118,7 +125,6 @@ int raw_import_new(
} }
*ret = TAKE_PTR(i); *ret = TAKE_PTR(i);
return 0; return 0;
} }
@ -127,13 +133,13 @@ static void raw_import_report_progress(RawImport *i) {
assert(i); assert(i);
/* We have no size information, unless the source is a regular file */ /* We have no size information, unless the source is a regular file */
if (!S_ISREG(i->st.st_mode)) if (!S_ISREG(i->input_stat.st_mode))
return; return;
if (i->written_compressed >= (uint64_t) i->st.st_size) if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
percent = 100; percent = 100;
else else
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
if (percent == i->last_percent) if (percent == i->last_percent)
return; return;
@ -149,11 +155,18 @@ static void raw_import_report_progress(RawImport *i) {
static int raw_import_maybe_convert_qcow2(RawImport *i) { static int raw_import_maybe_convert_qcow2(RawImport *i) {
_cleanup_close_ int converted_fd = -1; _cleanup_close_ int converted_fd = -1;
_cleanup_free_ char *t = NULL; _cleanup_(unlink_and_freep) char *t = NULL;
_cleanup_free_ char *f = NULL;
int r; int r;
assert(i); assert(i);
/* Do QCOW2 conversion if enabled and not in direct mode */
if ((i->flags & (IMPORT_CONVERT_QCOW2|IMPORT_DIRECT)) != IMPORT_CONVERT_QCOW2)
return 0;
assert(i->final_path);
r = qcow2_detect(i->output_fd); r = qcow2_detect(i->output_fd);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m"); return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
@ -161,26 +174,26 @@ static int raw_import_maybe_convert_qcow2(RawImport *i) {
return 0; return 0;
/* This is a QCOW2 image, let's convert it */ /* This is a QCOW2 image, let's convert it */
r = tempfn_random(i->final_path, NULL, &t); r = tempfn_random(i->final_path, NULL, &f);
if (r < 0) if (r < 0)
return log_oom(); return log_oom();
converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (converted_fd < 0) if (converted_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", t); return log_error_errno(errno, "Failed to create %s: %m", f);
t = TAKE_PTR(f);
(void) import_set_nocow_and_log(converted_fd, t); (void) import_set_nocow_and_log(converted_fd, t);
log_info("Unpacking QCOW2 file."); log_info("Unpacking QCOW2 file.");
r = qcow2_convert(i->output_fd, converted_fd); r = qcow2_convert(i->output_fd, converted_fd);
if (r < 0) { if (r < 0)
(void) unlink(t);
return log_error_errno(r, "Failed to convert qcow2 image: %m"); return log_error_errno(r, "Failed to convert qcow2 image: %m");
}
(void) unlink(i->temp_path); unlink_and_free(i->temp_path);
free_and_replace(i->temp_path, t); i->temp_path = TAKE_PTR(t);
CLOSE_AND_REPLACE(i->output_fd, converted_fd); CLOSE_AND_REPLACE(i->output_fd, converted_fd);
return 1; return 1;
@ -191,34 +204,45 @@ static int raw_import_finish(RawImport *i) {
assert(i); assert(i);
assert(i->output_fd >= 0); assert(i->output_fd >= 0);
assert(i->temp_path);
assert(i->final_path);
/* In case this was a sparse file, make sure the file system is right */ /* Nothing of what is below applies to block devices */
if (i->written_uncompressed > 0) { if (S_ISBLK(i->output_stat.st_mode)) {
if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
return log_error_errno(errno, "Failed to truncate file: %m"); if (i->flags & IMPORT_SYNC) {
if (fsync(i->output_fd) < 0)
return log_error_errno(errno, "Failed to synchronize block device: %m");
}
return 0;
} }
r = raw_import_maybe_convert_qcow2(i); assert(S_ISREG(i->output_stat.st_mode));
if (r < 0)
return r;
if (S_ISREG(i->st.st_mode)) { /* If an offset is specified we only are supposed to affect part of an existing output file or block
(void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME); * device, thus don't manipulate file properties in that case */
(void) copy_xattr(i->input_fd, i->output_fd, 0);
}
if (i->flags & IMPORT_READ_ONLY) { if (i->offset == UINT64_MAX) {
r = import_make_read_only_fd(i->output_fd); /* In case this was a sparse file, make sure the file size is right */
if (i->written_uncompressed > 0) {
if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
return log_error_errno(errno, "Failed to truncate file: %m");
}
r = raw_import_maybe_convert_qcow2(i);
if (r < 0) if (r < 0)
return r; return r;
if (S_ISREG(i->input_stat.st_mode)) {
(void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME);
(void) copy_xattr(i->input_fd, i->output_fd, 0);
}
} }
if (i->flags & IMPORT_FORCE) r = install_file(AT_FDCWD, i->temp_path ?: i->local,
(void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); AT_FDCWD, i->final_path,
(i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path); (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to move image into place: %m"); return log_error_errno(r, "Failed to move image into place: %m");
@ -231,26 +255,56 @@ static int raw_import_open_disk(RawImport *i) {
int r; int r;
assert(i); assert(i);
assert(i->local);
assert(!i->final_path); assert(!i->final_path);
assert(!i->temp_path); assert(!i->temp_path);
assert(i->output_fd < 0); assert(i->output_fd < 0);
i->final_path = strjoin(i->image_root, "/", i->local, ".raw"); if (i->flags & IMPORT_DIRECT) {
if (!i->final_path) (void) mkdir_parents_label(i->local, 0700);
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path); /* In direct mode we just open/create the local path and truncate it (like shell >
if (r < 0) * redirection would do it) — except if an offset was passed, in which case we are supposed
return log_oom(); * to operate on a section of the file only, in which case we apparently work on an some
* existing thing (i.e. are not the sole thing stored in the file), in which case we will
* neither truncate nor create. */
(void) mkdir_parents_label(i->temp_path, 0700); i->output_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
if (i->output_fd < 0)
return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); if (i->offset == UINT64_MAX)
if (i->output_fd < 0) (void) import_set_nocow_and_log(i->output_fd, i->local);
return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path); } else {
i->final_path = strjoin(i->image_root, "/", i->local, ".raw");
if (!i->final_path)
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0)
return log_oom();
(void) mkdir_parents_label(i->temp_path, 0700);
i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (i->output_fd < 0)
return log_error_errno(errno, "Failed to open destination '%s': %m", i->temp_path);
(void) import_set_nocow_and_log(i->output_fd, i->temp_path);
}
if (fstat(i->output_fd, &i->output_stat) < 0)
return log_error_errno(errno, "Failed to stat() output file: %m");
if (!S_ISREG(i->output_stat.st_mode) && !S_ISBLK(i->output_stat.st_mode))
return log_error_errno(SYNTHETIC_ERRNO(EBADFD),
"Target file is not a regular file or block device");
if (i->offset != UINT64_MAX) {
if (lseek(i->output_fd, i->offset, SEEK_SET) == (off_t) -1)
return log_error_errno(errno, "Failed to seek to offset: %m");
}
(void) import_set_nocow_and_log(i->output_fd, i->temp_path);
return 0; return 0;
} }
@ -265,7 +319,7 @@ static int raw_import_try_reflink(RawImport *i) {
if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED)
return 0; return 0;
if (!S_ISREG(i->st.st_mode)) if (!S_ISREG(i->input_stat.st_mode) || !S_ISREG(i->output_stat.st_mode))
return 0; return 0;
p = lseek(i->input_fd, 0, SEEK_CUR); p = lseek(i->input_fd, 0, SEEK_CUR);
@ -280,21 +334,57 @@ static int raw_import_try_reflink(RawImport *i) {
if (r >= 0) if (r >= 0)
return 1; return 1;
log_debug_errno(r, "Couldn't establish reflink, using copy: %m");
return 0; return 0;
} }
static int raw_import_write(const void *p, size_t sz, void *userdata) { static int raw_import_write(const void *p, size_t sz, void *userdata) {
RawImport *i = userdata; RawImport *i = userdata;
ssize_t n; bool too_much = false;
int r;
n = sparse_write(i->output_fd, p, sz, 64); assert(i);
if (n < 0) assert(p);
return (int) n; assert(sz > 0);
if ((size_t) n < sz)
return -EIO; if (i->written_uncompressed >= UINT64_MAX - sz)
return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
if (i->size_max != UINT64_MAX) {
if (i->written_uncompressed >= i->size_max) {
too_much = true;
goto finish;
}
if (i->written_uncompressed + sz > i->size_max) {
too_much = true;
sz = i->size_max - i->written_uncompressed; /* since we have the data in memory
* already, we might as well write it to
* disk to the max */
}
}
/* Generate sparse file if we created/truncated the file */
if (S_ISREG(i->output_stat.st_mode)) {
ssize_t n;
n = sparse_write(i->output_fd, p, sz, 64);
if (n < 0)
return log_error_errno((int) n, "Failed to write file: %m");
if ((size_t) n < sz)
return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
} else {
r = loop_write(i->output_fd, p, sz, false);
if (r < 0)
return log_error_errno(r, "Failed to write file: %m");
}
i->written_uncompressed += sz; i->written_uncompressed += sz;
finish:
if (too_much)
return log_error_errno(SYNTHETIC_ERRNO(E2BIG), "File too large");
return 0; return 0;
} }
@ -382,15 +472,22 @@ static int raw_import_on_defer(sd_event_source *s, void *userdata) {
return raw_import_process(i); return raw_import_process(i);
} }
int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) { int raw_import_start(
RawImport *i,
int fd,
const char *local,
uint64_t offset,
uint64_t size_max,
ImportFlags flags) {
int r; int r;
assert(i); assert(i);
assert(fd >= 0); assert(fd >= 0);
assert(local); assert(local);
assert(!(flags & ~IMPORT_FLAGS_MASK)); assert(!(flags & ~IMPORT_FLAGS_MASK_RAW));
assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
if (!hostname_is_valid(local, 0)) if (!import_validate_local(local, flags))
return -EINVAL; return -EINVAL;
if (i->input_fd >= 0) if (i->input_fd >= 0)
@ -405,8 +502,10 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
return r; return r;
i->flags = flags; i->flags = flags;
i->offset = offset;
i->size_max = size_max;
if (fstat(fd, &i->st) < 0) if (fstat(fd, &i->input_stat) < 0)
return -errno; return -errno;
r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i); r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i);
@ -422,5 +521,5 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
return r; return r;
i->input_fd = fd; i->input_fd = fd;
return r; return 0;
} }

2
src/import/import-raw.h
View File

@ -16,4 +16,4 @@ RawImport* raw_import_unref(RawImport *import);
DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref); DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref);
int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags); int raw_import_start(RawImport *i, int fd, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags);

95
src/import/import-tar.c
View File

@ -15,6 +15,7 @@
#include "import-common.h" #include "import-common.h"
#include "import-compress.h" #include "import-compress.h"
#include "import-tar.h" #include "import-tar.h"
#include "install-file.h"
#include "io-util.h" #include "io-util.h"
#include "machine-pool.h" #include "machine-pool.h"
#include "mkdir.h" #include "mkdir.h"
@ -54,7 +55,7 @@ struct TarImport {
uint64_t written_compressed; uint64_t written_compressed;
uint64_t written_uncompressed; uint64_t written_uncompressed;
struct stat st; struct stat input_stat;
pid_t tar_pid; pid_t tar_pid;
@ -136,13 +137,13 @@ static void tar_import_report_progress(TarImport *i) {
assert(i); assert(i);
/* We have no size information, unless the source is a regular file */ /* We have no size information, unless the source is a regular file */
if (!S_ISREG(i->st.st_mode)) if (!S_ISREG(i->input_stat.st_mode))
return; return;
if (i->written_compressed >= (uint64_t) i->st.st_size) if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
percent = 100; percent = 100;
else else
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size); percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
if (percent == i->last_percent) if (percent == i->last_percent)
return; return;
@ -157,12 +158,11 @@ static void tar_import_report_progress(TarImport *i) {
} }
static int tar_import_finish(TarImport *i) { static int tar_import_finish(TarImport *i) {
const char *d;
int r; int r;
assert(i); assert(i);
assert(i->tar_fd >= 0); assert(i->tar_fd >= 0);
assert(i->temp_path);
assert(i->final_path);
i->tar_fd = safe_close(i->tar_fd); i->tar_fd = safe_close(i->tar_fd);
@ -175,22 +175,20 @@ static int tar_import_finish(TarImport *i) {
return -EPROTO; return -EPROTO;
} }
r = import_mangle_os_tree(i->temp_path); assert_se(d = i->temp_path ?: i->local);
r = import_mangle_os_tree(d);
if (r < 0) if (r < 0)
return r; return r;
if (i->flags & IMPORT_READ_ONLY) { r = install_file(
r = import_make_read_only(i->temp_path); AT_FDCWD, d,
if (r < 0) AT_FDCWD, i->final_path,
return r; (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
} (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
if (i->flags & IMPORT_FORCE)
(void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to move image into place: %m"); return log_error_errno(r, "Failed to move '%s' into place: %m", i->final_path ?: i->local);
i->temp_path = mfree(i->temp_path); i->temp_path = mfree(i->temp_path);
@ -198,33 +196,54 @@ static int tar_import_finish(TarImport *i) {
} }
static int tar_import_fork_tar(TarImport *i) { static int tar_import_fork_tar(TarImport *i) {
const char *d, *root;
int r; int r;
assert(i); assert(i);
assert(i->local);
assert(!i->final_path); assert(!i->final_path);
assert(!i->temp_path); assert(!i->temp_path);
assert(i->tar_fd < 0); assert(i->tar_fd < 0);
i->final_path = path_join(i->image_root, i->local); if (i->flags & IMPORT_DIRECT) {
if (!i->final_path) d = i->local;
return log_oom(); root = NULL;
} else {
i->final_path = path_join(i->image_root, i->local);
if (!i->final_path)
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path); r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0) if (r < 0)
return log_oom(); return log_oom();
(void) mkdir_parents_label(i->temp_path, 0700); d = i->temp_path;
root = i->image_root;
r = btrfs_subvol_make_fallback(i->temp_path, 0755);
if (r < 0)
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
if (r > 0) { /* actually btrfs subvol */
(void) import_assign_pool_quota_and_warn(i->image_root);
(void) import_assign_pool_quota_and_warn(i->temp_path);
} }
i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid); assert(d);
(void) mkdir_parents_label(d, 0700);
if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE))
(void) rm_rf(d, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
if (i->flags & IMPORT_BTRFS_SUBVOL)
r = btrfs_subvol_make_fallback(d, 0755);
else
r = mkdir(d, 0755) < 0 ? -errno : 0;
if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
* because in that case our temporary path collided */
r = 0;
if (r < 0)
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", d);
if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */
if (!(i->flags & IMPORT_DIRECT))
(void) import_assign_pool_quota_and_warn(root);
(void) import_assign_pool_quota_and_warn(d);
}
i->tar_fd = import_fork_tar_x(d, &i->tar_pid);
if (i->tar_fd < 0) if (i->tar_fd < 0)
return i->tar_fd; return i->tar_fd;
@ -327,9 +346,9 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
assert(i); assert(i);
assert(fd >= 0); assert(fd >= 0);
assert(local); assert(local);
assert(!(flags & ~IMPORT_FLAGS_MASK)); assert(!(flags & ~IMPORT_FLAGS_MASK_TAR));
if (!hostname_is_valid(local, 0)) if (!import_validate_local(local, flags))
return -EINVAL; return -EINVAL;
if (i->input_fd >= 0) if (i->input_fd >= 0)
@ -345,7 +364,7 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
i->flags = flags; i->flags = flags;
if (fstat(fd, &i->st) < 0) if (fstat(fd, &i->input_stat) < 0)
return -errno; return -errno;
r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i); r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i);
@ -361,5 +380,5 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
return r; return r;
i->input_fd = fd; i->input_fd = fd;
return r; return 0;
} }

383
src/import/import.c
View File

@ -8,26 +8,114 @@
#include "alloc-util.h" #include "alloc-util.h"
#include "discover-image.h" #include "discover-image.h"
#include "env-util.h"
#include "fd-util.h" #include "fd-util.h"
#include "fs-util.h" #include "fs-util.h"
#include "hostname-util.h" #include "hostname-util.h"
#include "import-raw.h" #include "import-raw.h"
#include "import-tar.h" #include "import-tar.h"
#include "import-util.h" #include "import-util.h"
#include "io-util.h"
#include "main-func.h" #include "main-func.h"
#include "parse-argument.h"
#include "parse-util.h"
#include "signal-util.h" #include "signal-util.h"
#include "string-util.h" #include "string-util.h"
#include "terminal-util.h"
#include "verbs.h" #include "verbs.h"
static const char *arg_image_root = "/var/lib/machines"; static const char *arg_image_root = "/var/lib/machines";
static ImportFlags arg_import_flags = 0; static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { static int normalize_local(const char *local, char **ret) {
log_notice("Transfer aborted."); _cleanup_free_ char *ll = NULL;
sd_event_exit(sd_event_source_get_event(s), EINTR); int r;
assert(ret);
if (arg_import_flags & IMPORT_DIRECT) {
if (!local)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified.");
if (!path_is_absolute(local)) {
ll = path_join(arg_image_root, local);
if (!ll)
return log_oom();
local = ll;
}
if (!path_is_valid(local))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local path name '%s' is not valid.", local);
} else {
if (local) {
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
} else
local = "imported";
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
}
if (!ll) {
ll = strdup(local);
if (!ll)
return log_oom();
}
*ret = TAKE_PTR(ll);
return 0; return 0;
} }
static int open_source(const char *path, const char *local, int *ret_open_fd) {
_cleanup_close_ int open_fd = -1;
int retval;
assert(local);
assert(ret_open_fd);
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open raw image to import: %m");
retval = open_fd;
if (arg_offset != UINT64_MAX)
log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", path, arg_offset, local);
else
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
retval = STDIN_FILENO;
(void) fd_get_path(STDIN_FILENO, &pretty);
if (arg_offset != UINT64_MAX)
log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", strempty(pretty), arg_offset, local);
else
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
}
*ret_open_fd = TAKE_FD(open_fd);
return retval;
}
static void on_tar_finished(TarImport *import, int error, void *userdata) { static void on_tar_finished(TarImport *import, int error, void *userdata) {
sd_event *event = userdata; sd_event *event = userdata;
assert(import); assert(import);
@ -40,9 +128,9 @@ static void on_tar_finished(TarImport *import, int error, void *userdata) {
static int import_tar(int argc, char *argv[], void *userdata) { static int import_tar(int argc, char *argv[], void *userdata) {
_cleanup_(tar_import_unrefp) TarImport *import = NULL; _cleanup_(tar_import_unrefp) TarImport *import = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL;
const char *path = NULL, *local = NULL; const char *path = NULL, *local = NULL;
_cleanup_free_ char *ll = NULL;
_cleanup_close_ int open_fd = -1; _cleanup_close_ int open_fd = -1;
int r, fd; int r, fd;
@ -51,64 +139,44 @@ static int import_tar(int argc, char *argv[], void *userdata) {
if (argc >= 3) if (argc >= 3)
local = empty_or_dash_to_null(argv[2]); local = empty_or_dash_to_null(argv[2]);
else if (path) else if (path) {
local = basename(path); _cleanup_free_ char *l = NULL;
if (local) { r = path_extract_filename(path, &l);
r = tar_strip_suffixes(local, &ll); if (r < 0)
return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
r = tar_strip_suffixes(l, &ll);
if (r < 0) if (r < 0)
return log_oom(); return log_oom();
local = ll; local = ll;
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
} else
local = "imported";
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open tar image to import: %m");
fd = open_fd;
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDIN_FILENO;
(void) fd_get_path(fd, &pretty);
log_info("Importing '%s', saving as '%s'.", strna(pretty), local);
} }
r = sd_event_default(&event); r = normalize_local(local, &normalized);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m"); return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); fd = open_source(path, normalized, &open_fd);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); if (fd < 0)
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); return r;
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
log_info("File system synchronization on completion is off.");
r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to allocate importer: %m"); return log_error_errno(r, "Failed to allocate importer: %m");
r = tar_import_start(import, fd, local, arg_import_flags); r = tar_import_start(
import,
fd,
normalized,
arg_import_flags & IMPORT_FLAGS_MASK_TAR);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to import image: %m"); return log_error_errno(r, "Failed to import image: %m");
@ -132,9 +200,9 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
static int import_raw(int argc, char *argv[], void *userdata) { static int import_raw(int argc, char *argv[], void *userdata) {
_cleanup_(raw_import_unrefp) RawImport *import = NULL; _cleanup_(raw_import_unrefp) RawImport *import = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL; _cleanup_(sd_event_unrefp) sd_event *event = NULL;
const char *path = NULL, *local = NULL; const char *path = NULL, *local = NULL;
_cleanup_free_ char *ll = NULL;
_cleanup_close_ int open_fd = -1; _cleanup_close_ int open_fd = -1;
int r, fd; int r, fd;
@ -143,64 +211,46 @@ static int import_raw(int argc, char *argv[], void *userdata) {
if (argc >= 3) if (argc >= 3)
local = empty_or_dash_to_null(argv[2]); local = empty_or_dash_to_null(argv[2]);
else if (path) else if (path) {
local = basename(path); _cleanup_free_ char *l = NULL;
if (local) { r = path_extract_filename(path, &l);
r = raw_strip_suffixes(local, &ll); if (r < 0)
return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
r = raw_strip_suffixes(l, &ll);
if (r < 0) if (r < 0)
return log_oom(); return log_oom();
local = ll; local = ll;
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
} else
local = "imported";
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open raw image to import: %m");
fd = open_fd;
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDIN_FILENO;
(void) fd_get_path(fd, &pretty);
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
} }
r = sd_event_default(&event); r = normalize_local(local, &normalized);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m"); return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); fd = open_source(path, normalized, &open_fd);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); if (fd < 0)
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); return fd;
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
log_info("File system synchronization on completion is off.");
r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to allocate importer: %m"); return log_error_errno(r, "Failed to allocate importer: %m");
r = raw_import_start(import, fd, local, arg_import_flags); r = raw_import_start(
import,
fd,
normalized,
arg_offset,
arg_size_max,
arg_import_flags & IMPORT_FLAGS_MASK_RAW);
if (r < 0) if (r < 0)
return log_error_errno(r, "Failed to import image: %m"); return log_error_errno(r, "Failed to import image: %m");
@ -214,17 +264,32 @@ static int import_raw(int argc, char *argv[], void *userdata) {
static int help(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n" printf("%1$s [OPTIONS...] {COMMAND} ...\n"
"Import container or virtual machine images.\n\n" "\n%4$sImport container or virtual machine images.%5$s\n"
"\n%2$sCommands:%3$s\n"
" tar FILE [NAME] Import a TAR image\n"
" raw FILE [NAME] Import a RAW image\n"
"\n%2$sOptions:%3$s\n"
" -h --help Show this help\n" " -h --help Show this help\n"
" --version Show package version\n" " --version Show package version\n"
" --force Force creation of image\n" " --force Force creation of image\n"
" --image-root=PATH Image root directory\n" " --image-root=PATH Image root directory\n"
" --read-only Create a read-only image\n\n" " --read-only Create a read-only image\n"
"Commands:\n" " --direct Import directly to specified file\n"
" tar FILE [NAME] Import a TAR image\n" " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n"
" raw FILE [NAME] Import a RAW image\n", " instead of a directory\n"
program_invocation_short_name); " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n"
" subvolume\n"
" --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n"
" regular disk images\n"
" --sync=BOOL Controls whether to sync() before completing\n"
" --offset=BYTES Offset to seek to in destination\n"
" --size-max=BYTES Maximum number of bytes to write to destination\n",
program_invocation_short_name,
ansi_underline(),
ansi_normal(),
ansi_highlight(),
ansi_normal());
return 0; return 0;
} }
@ -236,6 +301,13 @@ static int parse_argv(int argc, char *argv[]) {
ARG_FORCE, ARG_FORCE,
ARG_IMAGE_ROOT, ARG_IMAGE_ROOT,
ARG_READ_ONLY, ARG_READ_ONLY,
ARG_DIRECT,
ARG_BTRFS_SUBVOL,
ARG_BTRFS_QUOTA,
ARG_CONVERT_QCOW2,
ARG_SYNC,
ARG_OFFSET,
ARG_SIZE_MAX,
}; };
static const struct option options[] = { static const struct option options[] = {
@ -244,10 +316,17 @@ static int parse_argv(int argc, char *argv[]) {
{ "force", no_argument, NULL, ARG_FORCE }, { "force", no_argument, NULL, ARG_FORCE },
{ "image-root", required_argument, NULL, ARG_IMAGE_ROOT }, { "image-root", required_argument, NULL, ARG_IMAGE_ROOT },
{ "read-only", no_argument, NULL, ARG_READ_ONLY }, { "read-only", no_argument, NULL, ARG_READ_ONLY },
{ "direct", no_argument, NULL, ARG_DIRECT },
{ "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL },
{ "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA },
{ "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 },
{ "sync", required_argument, NULL, ARG_SYNC },
{ "offset", required_argument, NULL, ARG_OFFSET },
{ "size-max", required_argument, NULL, ARG_SIZE_MAX },
{} {}
}; };
int c; int r, c;
assert(argc >= 0); assert(argc >= 0);
assert(argv); assert(argv);
@ -274,6 +353,68 @@ static int parse_argv(int argc, char *argv[]) {
arg_import_flags |= IMPORT_READ_ONLY; arg_import_flags |= IMPORT_READ_ONLY;
break; break;
case ARG_DIRECT:
arg_import_flags |= IMPORT_DIRECT;
break;
case ARG_BTRFS_SUBVOL:
r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
break;
case ARG_BTRFS_QUOTA:
r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
break;
case ARG_CONVERT_QCOW2:
r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r);
break;
case ARG_SYNC:
r = parse_boolean_argument("--sync=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
break;
case ARG_OFFSET: {
uint64_t u;
r = safe_atou64(optarg, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
arg_offset = u;
break;
}
case ARG_SIZE_MAX: {
uint64_t u;
r = parse_size(optarg, 1024, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
arg_size_max = u;
break;
}
case '?': case '?':
return -EINVAL; return -EINVAL;
@ -281,6 +422,15 @@ static int parse_argv(int argc, char *argv[]) {
assert_not_reached(); assert_not_reached();
} }
/* Make sure offset+size is still in the valid range if both set */
if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
((arg_size_max > (UINT64_MAX - arg_offset)) ||
!FILE_SIZE_VALID(arg_offset + arg_size_max)))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
return 1; return 1;
} }
@ -295,6 +445,31 @@ static int import_main(int argc, char *argv[]) {
return dispatch_verb(argc, argv, verbs, NULL); return dispatch_verb(argc, argv, verbs, NULL);
} }
static void parse_env(void) {
int r;
/* Let's make these relatively low-level settings also controllable via env vars. User can then set
* them to systemd-import if they like to tweak behaviour */
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
r = getenv_bool("SYSTEMD_IMPORT_SYNC");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
}
static int run(int argc, char *argv[]) { static int run(int argc, char *argv[]) {
int r; int r;
@ -302,9 +477,11 @@ static int run(int argc, char *argv[]) {
log_parse_environment(); log_parse_environment();
log_open(); log_open();
parse_env();
r = parse_argv(argc, argv); r = parse_argv(argc, argv);
if (r <= 0) if (r <= 0)
return 0; return r;
(void) ignore_signals(SIGPIPE); (void) ignore_signals(SIGPIPE);