shaba/systemd
1
0
Fork 0
mirror of https://github.com/systemd/systemd.git synced 2024-12-25 01:34:28 +03:00
Code

Merge pull request #20420 from poettering/import-beef-up

Browse Source 
import: modernizations, and various additions
This commit is contained in:
Lennart Poettering 2021-08-17 11:53:18 +02:00 committed by GitHub
commit 8b474a437c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 2217 additions and 887 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docs/ENVIRONMENT.md
View File

@ -316,3 +316,22 @@ fuzzers:
Note that is may be also useful to set `$SYSTEMD_LOG_LEVEL`, since all logging
is suppressed by default.
systemd-importd:
* `SYSTEMD_IMPORT_BTRFS_SUBVOL` – takes a boolean, which controls whether to
prefer creating btrfs subvolumes over plain directories for machine
images. Has no effect on non-btrfs file systems where subvolumes are not
available anyway. If not set, defaults to true.
* `SYSTEMD_IMPORT_BTRFS_QUOTA` – takes a boolean, which controls whether to set
up quota automatically for created btrfs subvolumes for machine images. If
not set, defaults to true. Has no effect if machines are placed in regular
directories, because btrfs subvolumes are not supported or disabled. If
enabled, the quota group of the subvolume is automatically added to a
combined quota group for all such machine subvolumes.
* `SYSTEMD_IMPORT_SYNC` – takes a boolean, which controls whether to
synchronize images to disk after installing them, before completing the
operation. If not set, defaults to true. If disabled installation of images
will be quicker, but not as safe.

87
src/import/import-common.c
View File

@ -13,6 +13,7 @@
#include "fd-util.h"
#include "fileio.h"
#include "fs-util.h"
#include "hostname-util.h"
#include "import-common.h"
#include "os-util.h"
#include "process-util.h"
@ -22,57 +23,6 @@
#include "tmpfile-util.h"
#include "util.h"
int import_make_read_only_fd(int fd) {
struct stat st;
int r;
assert(fd >= 0);
/* First, let's make this a read-only subvolume if it refers
* to a subvolume */
r = btrfs_subvol_set_read_only_fd(fd, true);
if (r >= 0)
return 0;
if (!ERRNO_IS_NOT_SUPPORTED(r) && !IN_SET(r, -ENOTDIR, -EINVAL))
return log_error_errno(r, "Failed to make subvolume read-only: %m");
/* This doesn't refer to a subvolume, or the file system isn't even btrfs. In that, case fall back to
* chmod()ing */
r = fstat(fd, &st);
if (r < 0)
return log_error_errno(errno, "Failed to stat image: %m");
if (S_ISDIR(st.st_mode)) {
/* For directories set the immutable flag on the dir itself */
r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL);
if (r < 0)
return log_error_errno(r, "Failed to set +i attribute on directory image: %m");
} else if (S_ISREG(st.st_mode)) {
/* For regular files drop "w" flags */
if ((st.st_mode & 0222) != 0)
if (fchmod(fd, st.st_mode & 07555) < 0)
return log_error_errno(errno, "Failed to chmod() image: %m");
} else
return log_error_errno(SYNTHETIC_ERRNO(EBADFD), "Image of unexpected type");
return 0;
}
int import_make_read_only(const char *path) {
_cleanup_close_ int fd = 1;
fd = open(path, O_RDONLY|O_NOCTTY|O_CLOEXEC);
if (fd < 0)
return log_error_errno(errno, "Failed to open %s: %m", path);
return import_make_read_only_fd(fd);
}
int import_fork_tar_x(const char *path, pid_t *ret) {
_cleanup_close_pair_ int pipefd[2] = { -1, -1 };
bool use_selinux;
@ -327,3 +277,38 @@ int import_mangle_os_tree(const char *path) {
return 0;
}
bool import_validate_local(const char *name, ImportFlags flags) {
/* By default we insist on a valid hostname for naming images. But optionally we relax that, in which
* case it can be any path name */
if (FLAGS_SET(flags, IMPORT_DIRECT))
return path_is_valid(name);
return hostname_is_valid(name, 0);
}
static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
log_notice("Transfer aborted.");
sd_event_exit(sd_event_source_get_event(s), EINTR);
return 0;
}
int import_allocate_event_with_signals(sd_event **ret) {
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
int r;
assert(ret);
r = sd_event_default(&event);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
*ret = TAKE_PTR(event);
return 0;
}

21
src/import/import-common.h
View File

@ -3,17 +3,26 @@
#include <sys/types.h>
#include "sd-event.h"
typedef enum ImportFlags {
IMPORT_FORCE = 1 << 0, /* replace existing image */
IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */
IMPORT_FORCE = 1 << 0, /* replace existing image */
IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */
IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */
IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
IMPORT_DIRECT = 1 << 5, /* import without rename games */
IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */
IMPORT_FLAGS_MASK = IMPORT_FORCE|IMPORT_READ_ONLY,
IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC,
IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC,
} ImportFlags;
int import_make_read_only_fd(int fd);
int import_make_read_only(const char *path);
int import_fork_tar_c(const char *path, pid_t *ret);
int import_fork_tar_x(const char *path, pid_t *ret);
int import_mangle_os_tree(const char *path);
bool import_validate_local(const char *name, ImportFlags flags);
int import_allocate_event_with_signals(sd_event **ret);

260
src/import/import-fs.c
View File

@ -12,15 +12,24 @@
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
#include "install-file.h"
#include "main-func.h"
#include "mkdir.h"
#include "parse-argument.h"
#include "ratelimit.h"
#include "rm-rf.h"
#include "signal-util.h"
#include "string-util.h"
#include "terminal-util.h"
#include "tmpfile-util.h"
#include "verbs.h"
static bool arg_force = false;
static bool arg_read_only = false;
static bool arg_btrfs_subvol = true;
static bool arg_btrfs_quota = true;
static bool arg_sync = true;
static bool arg_direct = false;
static const char *arg_image_root = "/var/lib/machines";
typedef struct ProgressInfo {
@ -31,12 +40,6 @@ typedef struct ProgressInfo {
bool logged_incomplete;
} ProgressInfo;
static volatile sig_atomic_t cancelled = false;
static void sigterm_sigint(int sig) {
cancelled = true;
}
static void progress_info_free(ProgressInfo *p) {
free(p->path);
}
@ -56,7 +59,7 @@ static void progress_show(ProgressInfo *p) {
/* Mention the list is incomplete before showing first output. */
if (!p->logged_incomplete) {
log_notice("(Note, file list shown below is incomplete, and is intended as sporadic progress report only.)");
log_notice("(Note: file list shown below is incomplete, and is intended as sporadic progress report only.)");
p->logged_incomplete = true;
}
@ -72,9 +75,6 @@ static int progress_path(const char *path, const struct stat *st, void *userdata
assert(p);
if (cancelled)
return -EOWNERDEAD;
r = free_and_strdup(&p->path, path);
if (r < 0)
return r;
@ -91,9 +91,6 @@ static int progress_bytes(uint64_t nbytes, void *userdata) {
assert(p);
assert(p->size != UINT64_MAX);
if (cancelled)
return -EOWNERDEAD;
p->size += nbytes;
progress_show(p);
@ -103,44 +100,60 @@ static int progress_bytes(uint64_t nbytes, void *userdata) {
static int import_fs(int argc, char *argv[], void *userdata) {
_cleanup_(rm_rf_subvolume_and_freep) char *temp_path = NULL;
_cleanup_(progress_info_free) ProgressInfo progress = {};
const char *path = NULL, *local = NULL, *final_path;
_cleanup_free_ char *l = NULL, *final_path = NULL;
const char *path = NULL, *local = NULL, *dest = NULL;
_cleanup_close_ int open_fd = -1;
struct sigaction old_sigint_sa, old_sigterm_sa;
static const struct sigaction sa = {
.sa_handler = sigterm_sigint,
.sa_flags = SA_RESTART,
};
int r, fd;
if (argc >= 2)
path = argv[1];
path = empty_or_dash_to_null(path);
path = empty_or_dash_to_null(argv[1]);
if (argc >= 3)
local = argv[2];
else if (path)
local = basename(path);
local = empty_or_dash_to_null(local);
local = empty_or_dash_to_null(argv[2]);
else if (path) {
r = path_extract_filename(path, &l);
if (r < 0)
return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
if (local) {
if (!hostname_is_valid(local, 0))
local = l;
}
if (arg_direct) {
if (!local)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified.");
if (path_is_absolute(local))
final_path = strdup(local);
else
final_path = path_join(arg_image_root, local);
if (!final_path)
return log_oom();
if (!path_is_valid(final_path))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
"Local path name '%s' is not valid.", final_path);
} else {
if (local) {
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.", local);
} else
local = "imported";
final_path = path_join(arg_image_root, local);
if (!final_path)
return log_oom();
if (!arg_force) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else {
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
"Image '%s' already exists.", local);
}
} else
local = "imported";
}
if (path) {
open_fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
@ -159,84 +172,107 @@ static int import_fs(int argc, char *argv[], void *userdata) {
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
}
final_path = prefix_roota(arg_image_root, local);
if (!arg_sync)
log_info("File system synchronization on completion is off.");
r = tempfn_random(final_path, NULL, &temp_path);
if (r < 0)
return log_oom();
if (arg_direct) {
if (arg_force)
(void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
(void) mkdir_parents_label(temp_path, 0700);
dest = final_path;
} else {
r = tempfn_random(final_path, NULL, &temp_path);
if (r < 0)
return log_oom();
(void) mkdir_parents_label(temp_path, 0700);
dest = temp_path;
}
progress.limit = (RateLimit) { 200*USEC_PER_MSEC, 1 };
/* Hook into SIGINT/SIGTERM, so that we can cancel things then */
assert_se(sigaction(SIGINT, &sa, &old_sigint_sa) >= 0);
assert_se(sigaction(SIGTERM, &sa, &old_sigterm_sa) >= 0);
{
BLOCK_SIGNALS(SIGINT, SIGTERM);
r = btrfs_subvol_snapshot_fd_full(
fd,
temp_path,
BTRFS_SNAPSHOT_FALLBACK_COPY|BTRFS_SNAPSHOT_RECURSIVE|BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|BTRFS_SNAPSHOT_QUOTA,
progress_path,
progress_bytes,
&progress);
if (r == -EOWNERDEAD) { /* SIGINT + SIGTERM cause this, see signal handler above */
log_error("Copy cancelled.");
goto finish;
}
if (r < 0) {
log_error_errno(r, "Failed to copy directory: %m");
goto finish;
if (arg_btrfs_subvol)
r = btrfs_subvol_snapshot_fd_full(
fd,
dest,
BTRFS_SNAPSHOT_FALLBACK_COPY|
BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
BTRFS_SNAPSHOT_RECURSIVE|
BTRFS_SNAPSHOT_SIGINT|
BTRFS_SNAPSHOT_SIGTERM,
progress_path,
progress_bytes,
&progress);
else
r = copy_directory_fd_full(
fd,
dest,
COPY_REFLINK|
COPY_SAME_MOUNT|
COPY_HARDLINKS|
COPY_SIGINT|
COPY_SIGTERM|
(arg_direct ? COPY_MERGE_EMPTY : 0),
progress_path,
progress_bytes,
&progress);
if (r == -EINTR) /* SIGINT/SIGTERM hit */
return log_error_errno(r, "Copy cancelled.");
if (r < 0)
return log_error_errno(r, "Failed to copy directory: %m");
}
r = import_mangle_os_tree(temp_path);
r = import_mangle_os_tree(dest);
if (r < 0)
goto finish;
return r;
(void) import_assign_pool_quota_and_warn(arg_image_root);
(void) import_assign_pool_quota_and_warn(temp_path);
if (arg_read_only) {
r = import_make_read_only(temp_path);
if (r < 0) {
log_error_errno(r, "Failed to make directory read-only: %m");
goto finish;
}
if (arg_btrfs_quota) {
if (!arg_direct)
(void) import_assign_pool_quota_and_warn(arg_image_root);
(void) import_assign_pool_quota_and_warn(dest);
}
if (arg_force)
(void) rm_rf(final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = rename_noreplace(AT_FDCWD, temp_path, AT_FDCWD, final_path);
if (r < 0) {
log_error_errno(r, "Failed to move image into place: %m");
goto finish;
}
r = install_file(AT_FDCWD, dest,
AT_FDCWD, arg_direct ? NULL : final_path, /* pass NULL as target in case of direct
* mode since file is already in place */
(arg_force ? INSTALL_REPLACE : 0) |
(arg_read_only ? INSTALL_READ_ONLY : 0) |
(arg_sync ? INSTALL_SYNCFS : 0));
if (r < 0)
return log_error_errno(r, "Failed install directory as '%s': %m", final_path);
temp_path = mfree(temp_path);
log_info("Exiting.");
finish:
/* Put old signal handlers into place */
assert_se(sigaction(SIGINT, &old_sigint_sa, NULL) >= 0);
assert_se(sigaction(SIGTERM, &old_sigterm_sa, NULL) >= 0);
log_info("Directory '%s successfully installed. Exiting.", final_path);
return 0;
}
static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n"
"Import container images from a file system.\n\n"
printf("%1$s [OPTIONS...] {COMMAND} ...\n"
"\n%4$sImport container images from a file system directories.%5$s\n"
"\n%2$sCommands:%3$s\n"
" run DIRECTORY [NAME] Import a directory\n"
"\n%2$sOptions:%3$s\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n"
" --image-root=PATH Image root directory\n"
" --read-only Create a read-only image\n\n"
"Commands:\n"
" run DIRECTORY [NAME] Import a directory\n",
program_invocation_short_name);
" --read-only Create a read-only image\n"
" --direct Import directly to specified directory\n"
" --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n"
" instead of a directory\n"
" --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n"
" subvolume\n"
" --sync=BOOL Controls whether to sync() before completing\n",
program_invocation_short_name,
ansi_underline(),
ansi_normal(),
ansi_highlight(),
ansi_normal());
return 0;
}
@ -248,6 +284,10 @@ static int parse_argv(int argc, char *argv[]) {
ARG_FORCE,
ARG_IMAGE_ROOT,
ARG_READ_ONLY,
ARG_DIRECT,
ARG_BTRFS_SUBVOL,
ARG_BTRFS_QUOTA,
ARG_SYNC,
};
static const struct option options[] = {
@ -256,10 +296,14 @@ static int parse_argv(int argc, char *argv[]) {
{ "force", no_argument, NULL, ARG_FORCE },
{ "image-root", required_argument, NULL, ARG_IMAGE_ROOT },
{ "read-only", no_argument, NULL, ARG_READ_ONLY },
{ "direct", no_argument, NULL, ARG_DIRECT },
{ "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL },
{ "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA },
{ "sync", required_argument, NULL, ARG_SYNC },
{}
};
int c;
int c, r;
assert(argc >= 0);
assert(argv);
@ -286,6 +330,31 @@ static int parse_argv(int argc, char *argv[]) {
arg_read_only = true;
break;
case ARG_DIRECT:
arg_direct = true;
break;
case ARG_BTRFS_SUBVOL:
r = parse_boolean_argument("--btrfs-subvol=", optarg, &arg_btrfs_subvol);
if (r < 0)
return r;
break;
case ARG_BTRFS_QUOTA:
r = parse_boolean_argument("--btrfs-quota=", optarg, &arg_btrfs_quota);
if (r < 0)
return r;
break;
case ARG_SYNC:
r = parse_boolean_argument("--sync=", optarg, &arg_sync);
if (r < 0)
return r;
break;
case '?':
return -EINVAL;
@ -307,7 +376,7 @@ static int import_fs_main(int argc, char *argv[]) {
return dispatch_verb(argc, argv, verbs, NULL);
}
int main(int argc, char *argv[]) {
static int run(int argc, char *argv[]) {
int r;
setlocale(LC_ALL, "");
@ -316,10 +385,9 @@ int main(int argc, char *argv[]) {
r = parse_argv(argc, argv);
if (r <= 0)
goto finish;
return r;
r = import_fs_main(argc, argv);
finish:
return r < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
return import_fs_main(argc, argv);
}
DEFINE_MAIN_FUNCTION(run);

217
src/import/import-raw.c
View File

@ -14,6 +14,7 @@
#include "import-common.h"
#include "import-compress.h"
#include "import-raw.h"
#include "install-file.h"
#include "io-util.h"
#include "machine-pool.h"
#include "mkdir.h"
@ -52,23 +53,27 @@ struct RawImport {
uint64_t written_compressed;
uint64_t written_uncompressed;
struct stat st;
struct stat input_stat;
struct stat output_stat;
unsigned last_percent;
RateLimit progress_ratelimit;
uint64_t offset;
uint64_t size_max;
};
RawImport* raw_import_unref(RawImport *i) {
if (!i)
return NULL;
sd_event_unref(i->event);
sd_event_source_unref(i->input_event_source);
unlink_and_free(i->temp_path);
import_compress_free(&i->compress);
sd_event_source_unref(i->input_event_source);
sd_event_unref(i->event);
safe_close(i->output_fd);
@ -107,6 +112,8 @@ int raw_import_new(
.last_percent = UINT_MAX,
.image_root = TAKE_PTR(root),
.progress_ratelimit = { 100 * USEC_PER_MSEC, 1 },
.offset = UINT64_MAX,
.size_max = UINT64_MAX,
};
if (event)
@ -118,7 +125,6 @@ int raw_import_new(
}
*ret = TAKE_PTR(i);
return 0;
}
@ -127,13 +133,13 @@ static void raw_import_report_progress(RawImport *i) {
assert(i);
/* We have no size information, unless the source is a regular file */
if (!S_ISREG(i->st.st_mode))
if (!S_ISREG(i->input_stat.st_mode))
return;
if (i->written_compressed >= (uint64_t) i->st.st_size)
if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
percent = 100;
else
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size);
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
if (percent == i->last_percent)
return;
@ -149,11 +155,18 @@ static void raw_import_report_progress(RawImport *i) {
static int raw_import_maybe_convert_qcow2(RawImport *i) {
_cleanup_close_ int converted_fd = -1;
_cleanup_free_ char *t = NULL;
_cleanup_(unlink_and_freep) char *t = NULL;
_cleanup_free_ char *f = NULL;
int r;
assert(i);
/* Do QCOW2 conversion if enabled and not in direct mode */
if ((i->flags & (IMPORT_CONVERT_QCOW2|IMPORT_DIRECT)) != IMPORT_CONVERT_QCOW2)
return 0;
assert(i->final_path);
r = qcow2_detect(i->output_fd);
if (r < 0)
return log_error_errno(r, "Failed to detect whether this is a QCOW2 image: %m");
@ -161,26 +174,26 @@ static int raw_import_maybe_convert_qcow2(RawImport *i) {
return 0;
/* This is a QCOW2 image, let's convert it */
r = tempfn_random(i->final_path, NULL, &t);
r = tempfn_random(i->final_path, NULL, &f);
if (r < 0)
return log_oom();
converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (converted_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", t);
return log_error_errno(errno, "Failed to create %s: %m", f);
t = TAKE_PTR(f);
(void) import_set_nocow_and_log(converted_fd, t);
log_info("Unpacking QCOW2 file.");
r = qcow2_convert(i->output_fd, converted_fd);
if (r < 0) {
(void) unlink(t);
if (r < 0)
return log_error_errno(r, "Failed to convert qcow2 image: %m");
}
(void) unlink(i->temp_path);
free_and_replace(i->temp_path, t);
unlink_and_free(i->temp_path);
i->temp_path = TAKE_PTR(t);
CLOSE_AND_REPLACE(i->output_fd, converted_fd);
return 1;
@ -191,34 +204,45 @@ static int raw_import_finish(RawImport *i) {
assert(i);
assert(i->output_fd >= 0);
assert(i->temp_path);
assert(i->final_path);
/* In case this was a sparse file, make sure the file system is right */
if (i->written_uncompressed > 0) {
if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
return log_error_errno(errno, "Failed to truncate file: %m");
/* Nothing of what is below applies to block devices */
if (S_ISBLK(i->output_stat.st_mode)) {
if (i->flags & IMPORT_SYNC) {
if (fsync(i->output_fd) < 0)
return log_error_errno(errno, "Failed to synchronize block device: %m");
}
return 0;
}
r = raw_import_maybe_convert_qcow2(i);
if (r < 0)
return r;
assert(S_ISREG(i->output_stat.st_mode));
if (S_ISREG(i->st.st_mode)) {
(void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME);
(void) copy_xattr(i->input_fd, i->output_fd, 0);
}
/* If an offset is specified we only are supposed to affect part of an existing output file or block
* device, thus don't manipulate file properties in that case */
if (i->flags & IMPORT_READ_ONLY) {
r = import_make_read_only_fd(i->output_fd);
if (i->offset == UINT64_MAX) {
/* In case this was a sparse file, make sure the file size is right */
if (i->written_uncompressed > 0) {
if (ftruncate(i->output_fd, i->written_uncompressed) < 0)
return log_error_errno(errno, "Failed to truncate file: %m");
}
r = raw_import_maybe_convert_qcow2(i);
if (r < 0)
return r;
if (S_ISREG(i->input_stat.st_mode)) {
(void) copy_times(i->input_fd, i->output_fd, COPY_CRTIME);
(void) copy_xattr(i->input_fd, i->output_fd, 0);
}
}
if (i->flags & IMPORT_FORCE)
(void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
r = install_file(AT_FDCWD, i->temp_path ?: i->local,
AT_FDCWD, i->final_path,
(i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
(i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0)
return log_error_errno(r, "Failed to move image into place: %m");
@ -231,26 +255,56 @@ static int raw_import_open_disk(RawImport *i) {
int r;
assert(i);
assert(i->local);
assert(!i->final_path);
assert(!i->temp_path);
assert(i->output_fd < 0);
i->final_path = strjoin(i->image_root, "/", i->local, ".raw");
if (!i->final_path)
return log_oom();
if (i->flags & IMPORT_DIRECT) {
(void) mkdir_parents_label(i->local, 0700);
r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0)
return log_oom();
/* In direct mode we just open/create the local path and truncate it (like shell >
* redirection would do it) — except if an offset was passed, in which case we are supposed
* to operate on a section of the file only, in which case we apparently work on an some
* existing thing (i.e. are not the sole thing stored in the file), in which case we will
* neither truncate nor create. */
(void) mkdir_parents_label(i->temp_path, 0700);
i->output_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
if (i->output_fd < 0)
return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (i->output_fd < 0)
return log_error_errno(errno, "Failed to open destination %s: %m", i->temp_path);
if (i->offset == UINT64_MAX)
(void) import_set_nocow_and_log(i->output_fd, i->local);
} else {
i->final_path = strjoin(i->image_root, "/", i->local, ".raw");
if (!i->final_path)
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0)
return log_oom();
(void) mkdir_parents_label(i->temp_path, 0700);
i->output_fd = open(i->temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (i->output_fd < 0)
return log_error_errno(errno, "Failed to open destination '%s': %m", i->temp_path);
(void) import_set_nocow_and_log(i->output_fd, i->temp_path);
}
if (fstat(i->output_fd, &i->output_stat) < 0)
return log_error_errno(errno, "Failed to stat() output file: %m");
if (!S_ISREG(i->output_stat.st_mode) && !S_ISBLK(i->output_stat.st_mode))
return log_error_errno(SYNTHETIC_ERRNO(EBADFD),
"Target file is not a regular file or block device");
if (i->offset != UINT64_MAX) {
if (lseek(i->output_fd, i->offset, SEEK_SET) == (off_t) -1)
return log_error_errno(errno, "Failed to seek to offset: %m");
}
(void) import_set_nocow_and_log(i->output_fd, i->temp_path);
return 0;
}
@ -265,7 +319,7 @@ static int raw_import_try_reflink(RawImport *i) {
if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED)
return 0;
if (!S_ISREG(i->st.st_mode))
if (!S_ISREG(i->input_stat.st_mode) || !S_ISREG(i->output_stat.st_mode))
return 0;
p = lseek(i->input_fd, 0, SEEK_CUR);
@ -280,21 +334,57 @@ static int raw_import_try_reflink(RawImport *i) {
if (r >= 0)
return 1;
log_debug_errno(r, "Couldn't establish reflink, using copy: %m");
return 0;
}
static int raw_import_write(const void *p, size_t sz, void *userdata) {
RawImport *i = userdata;
ssize_t n;
bool too_much = false;
int r;
n = sparse_write(i->output_fd, p, sz, 64);
if (n < 0)
return (int) n;
if ((size_t) n < sz)
return -EIO;
assert(i);
assert(p);
assert(sz > 0);
if (i->written_uncompressed >= UINT64_MAX - sz)
return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
if (i->size_max != UINT64_MAX) {
if (i->written_uncompressed >= i->size_max) {
too_much = true;
goto finish;
}
if (i->written_uncompressed + sz > i->size_max) {
too_much = true;
sz = i->size_max - i->written_uncompressed; /* since we have the data in memory
* already, we might as well write it to
* disk to the max */
}
}
/* Generate sparse file if we created/truncated the file */
if (S_ISREG(i->output_stat.st_mode)) {
ssize_t n;
n = sparse_write(i->output_fd, p, sz, 64);
if (n < 0)
return log_error_errno((int) n, "Failed to write file: %m");
if ((size_t) n < sz)
return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
} else {
r = loop_write(i->output_fd, p, sz, false);
if (r < 0)
return log_error_errno(r, "Failed to write file: %m");
}
i->written_uncompressed += sz;
finish:
if (too_much)
return log_error_errno(SYNTHETIC_ERRNO(E2BIG), "File too large");
return 0;
}
@ -382,15 +472,22 @@ static int raw_import_on_defer(sd_event_source *s, void *userdata) {
return raw_import_process(i);
}
int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags) {
int raw_import_start(
RawImport *i,
int fd,
const char *local,
uint64_t offset,
uint64_t size_max,
ImportFlags flags) {
int r;
assert(i);
assert(fd >= 0);
assert(local);
assert(!(flags & ~IMPORT_FLAGS_MASK));
assert(!(flags & ~IMPORT_FLAGS_MASK_RAW));
assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
if (!hostname_is_valid(local, 0))
if (!import_validate_local(local, flags))
return -EINVAL;
if (i->input_fd >= 0)
@ -405,8 +502,10 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
return r;
i->flags = flags;
i->offset = offset;
i->size_max = size_max;
if (fstat(fd, &i->st) < 0)
if (fstat(fd, &i->input_stat) < 0)
return -errno;
r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, raw_import_on_input, i);
@ -422,5 +521,5 @@ int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags)
return r;
i->input_fd = fd;
return r;
return 0;
}

2
src/import/import-raw.h
View File

@ -16,4 +16,4 @@ RawImport* raw_import_unref(RawImport *import);
DEFINE_TRIVIAL_CLEANUP_FUNC(RawImport*, raw_import_unref);
int raw_import_start(RawImport *i, int fd, const char *local, ImportFlags flags);
int raw_import_start(RawImport *i, int fd, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags);

95
src/import/import-tar.c
View File

@ -15,6 +15,7 @@
#include "import-common.h"
#include "import-compress.h"
#include "import-tar.h"
#include "install-file.h"
#include "io-util.h"
#include "machine-pool.h"
#include "mkdir.h"
@ -54,7 +55,7 @@ struct TarImport {
uint64_t written_compressed;
uint64_t written_uncompressed;
struct stat st;
struct stat input_stat;
pid_t tar_pid;
@ -136,13 +137,13 @@ static void tar_import_report_progress(TarImport *i) {
assert(i);
/* We have no size information, unless the source is a regular file */
if (!S_ISREG(i->st.st_mode))
if (!S_ISREG(i->input_stat.st_mode))
return;
if (i->written_compressed >= (uint64_t) i->st.st_size)
if (i->written_compressed >= (uint64_t) i->input_stat.st_size)
percent = 100;
else
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->st.st_size);
percent = (unsigned) ((i->written_compressed * UINT64_C(100)) / (uint64_t) i->input_stat.st_size);
if (percent == i->last_percent)
return;
@ -157,12 +158,11 @@ static void tar_import_report_progress(TarImport *i) {
}
static int tar_import_finish(TarImport *i) {
const char *d;
int r;
assert(i);
assert(i->tar_fd >= 0);
assert(i->temp_path);
assert(i->final_path);
i->tar_fd = safe_close(i->tar_fd);
@ -175,22 +175,20 @@ static int tar_import_finish(TarImport *i) {
return -EPROTO;
}
r = import_mangle_os_tree(i->temp_path);
assert_se(d = i->temp_path ?: i->local);
r = import_mangle_os_tree(d);
if (r < 0)
return r;
if (i->flags & IMPORT_READ_ONLY) {
r = import_make_read_only(i->temp_path);
if (r < 0)
return r;
}
if (i->flags & IMPORT_FORCE)
(void) rm_rf(i->final_path, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
r = install_file(
AT_FDCWD, d,
AT_FDCWD, i->final_path,
(i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) |
(i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0)
return log_error_errno(r, "Failed to move image into place: %m");
return log_error_errno(r, "Failed to move '%s' into place: %m", i->final_path ?: i->local);
i->temp_path = mfree(i->temp_path);
@ -198,33 +196,54 @@ static int tar_import_finish(TarImport *i) {
}
static int tar_import_fork_tar(TarImport *i) {
const char *d, *root;
int r;
assert(i);
assert(i->local);
assert(!i->final_path);
assert(!i->temp_path);
assert(i->tar_fd < 0);
i->final_path = path_join(i->image_root, i->local);
if (!i->final_path)
return log_oom();
if (i->flags & IMPORT_DIRECT) {
d = i->local;
root = NULL;
} else {
i->final_path = path_join(i->image_root, i->local);
if (!i->final_path)
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0)
return log_oom();
r = tempfn_random(i->final_path, NULL, &i->temp_path);
if (r < 0)
return log_oom();
(void) mkdir_parents_label(i->temp_path, 0700);
r = btrfs_subvol_make_fallback(i->temp_path, 0755);
if (r < 0)
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
if (r > 0) { /* actually btrfs subvol */
(void) import_assign_pool_quota_and_warn(i->image_root);
(void) import_assign_pool_quota_and_warn(i->temp_path);
d = i->temp_path;
root = i->image_root;
}
i->tar_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
assert(d);
(void) mkdir_parents_label(d, 0700);
if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE))
(void) rm_rf(d, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
if (i->flags & IMPORT_BTRFS_SUBVOL)
r = btrfs_subvol_make_fallback(d, 0755);
else
r = mkdir(d, 0755) < 0 ? -errno : 0;
if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
* because in that case our temporary path collided */
r = 0;
if (r < 0)
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", d);
if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */
if (!(i->flags & IMPORT_DIRECT))
(void) import_assign_pool_quota_and_warn(root);
(void) import_assign_pool_quota_and_warn(d);
}
i->tar_fd = import_fork_tar_x(d, &i->tar_pid);
if (i->tar_fd < 0)
return i->tar_fd;
@ -327,9 +346,9 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
assert(i);
assert(fd >= 0);
assert(local);
assert(!(flags & ~IMPORT_FLAGS_MASK));
assert(!(flags & ~IMPORT_FLAGS_MASK_TAR));
if (!hostname_is_valid(local, 0))
if (!import_validate_local(local, flags))
return -EINVAL;
if (i->input_fd >= 0)
@ -345,7 +364,7 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
i->flags = flags;
if (fstat(fd, &i->st) < 0)
if (fstat(fd, &i->input_stat) < 0)
return -errno;
r = sd_event_add_io(i->event, &i->input_event_source, fd, EPOLLIN, tar_import_on_input, i);
@ -361,5 +380,5 @@ int tar_import_start(TarImport *i, int fd, const char *local, ImportFlags flags)
return r;
i->input_fd = fd;
return r;
return 0;
}

383
src/import/import.c
View File

@ -8,26 +8,114 @@
#include "alloc-util.h"
#include "discover-image.h"
#include "env-util.h"
#include "fd-util.h"
#include "fs-util.h"
#include "hostname-util.h"
#include "import-raw.h"
#include "import-tar.h"
#include "import-util.h"
#include "io-util.h"
#include "main-func.h"
#include "parse-argument.h"
#include "parse-util.h"
#include "signal-util.h"
#include "string-util.h"
#include "terminal-util.h"
#include "verbs.h"
static const char *arg_image_root = "/var/lib/machines";
static ImportFlags arg_import_flags = 0;
static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
log_notice("Transfer aborted.");
sd_event_exit(sd_event_source_get_event(s), EINTR);
static int normalize_local(const char *local, char **ret) {
_cleanup_free_ char *ll = NULL;
int r;
assert(ret);
if (arg_import_flags & IMPORT_DIRECT) {
if (!local)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local path specified.");
if (!path_is_absolute(local)) {
ll = path_join(arg_image_root, local);
if (!ll)
return log_oom();
local = ll;
}
if (!path_is_valid(local))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local path name '%s' is not valid.", local);
} else {
if (local) {
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
} else
local = "imported";
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
}
if (!ll) {
ll = strdup(local);
if (!ll)
return log_oom();
}
*ret = TAKE_PTR(ll);
return 0;
}
static int open_source(const char *path, const char *local, int *ret_open_fd) {
_cleanup_close_ int open_fd = -1;
int retval;
assert(local);
assert(ret_open_fd);
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open raw image to import: %m");
retval = open_fd;
if (arg_offset != UINT64_MAX)
log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", path, arg_offset, local);
else
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
retval = STDIN_FILENO;
(void) fd_get_path(STDIN_FILENO, &pretty);
if (arg_offset != UINT64_MAX)
log_info("Importing '%s', saving at offset %" PRIu64 " in '%s'.", strempty(pretty), arg_offset, local);
else
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
}
*ret_open_fd = TAKE_FD(open_fd);
return retval;
}
static void on_tar_finished(TarImport *import, int error, void *userdata) {
sd_event *event = userdata;
assert(import);
@ -40,9 +128,9 @@ static void on_tar_finished(TarImport *import, int error, void *userdata) {
static int import_tar(int argc, char *argv[], void *userdata) {
_cleanup_(tar_import_unrefp) TarImport *import = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
const char *path = NULL, *local = NULL;
_cleanup_free_ char *ll = NULL;
_cleanup_close_ int open_fd = -1;
int r, fd;
@ -51,64 +139,44 @@ static int import_tar(int argc, char *argv[], void *userdata) {
if (argc >= 3)
local = empty_or_dash_to_null(argv[2]);
else if (path)
local = basename(path);
else if (path) {
_cleanup_free_ char *l = NULL;
if (local) {
r = tar_strip_suffixes(local, &ll);
r = path_extract_filename(path, &l);
if (r < 0)
return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
r = tar_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
} else
local = "imported";
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open tar image to import: %m");
fd = open_fd;
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDIN_FILENO;
(void) fd_get_path(fd, &pretty);
log_info("Importing '%s', saving as '%s'.", strna(pretty), local);
}
r = sd_event_default(&event);
r = normalize_local(local, &normalized);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
fd = open_source(path, normalized, &open_fd);
if (fd < 0)
return r;
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
log_info("File system synchronization on completion is off.");
r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate importer: %m");
r = tar_import_start(import, fd, local, arg_import_flags);
r = tar_import_start(
import,
fd,
normalized,
arg_import_flags & IMPORT_FLAGS_MASK_TAR);
if (r < 0)
return log_error_errno(r, "Failed to import image: %m");
@ -132,9 +200,9 @@ static void on_raw_finished(RawImport *import, int error, void *userdata) {
static int import_raw(int argc, char *argv[], void *userdata) {
_cleanup_(raw_import_unrefp) RawImport *import = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
const char *path = NULL, *local = NULL;
_cleanup_free_ char *ll = NULL;
_cleanup_close_ int open_fd = -1;
int r, fd;
@ -143,64 +211,46 @@ static int import_raw(int argc, char *argv[], void *userdata) {
if (argc >= 3)
local = empty_or_dash_to_null(argv[2]);
else if (path)
local = basename(path);
else if (path) {
_cleanup_free_ char *l = NULL;
if (local) {
r = raw_strip_suffixes(local, &ll);
r = path_extract_filename(path, &l);
if (r < 0)
return log_error_errno(r, "Failed to extract filename from path '%s': %m", path);
r = raw_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
} else
local = "imported";
if (path) {
open_fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
if (open_fd < 0)
return log_error_errno(errno, "Failed to open raw image to import: %m");
fd = open_fd;
log_info("Importing '%s', saving as '%s'.", path, local);
} else {
_cleanup_free_ char *pretty = NULL;
fd = STDIN_FILENO;
(void) fd_get_path(fd, &pretty);
log_info("Importing '%s', saving as '%s'.", strempty(pretty), local);
}
r = sd_event_default(&event);
r = normalize_local(local, &normalized);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
fd = open_source(path, normalized, &open_fd);
if (fd < 0)
return fd;
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC))
log_info("File system synchronization on completion is off.");
r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate importer: %m");
r = raw_import_start(import, fd, local, arg_import_flags);
r = raw_import_start(
import,
fd,
normalized,
arg_offset,
arg_size_max,
arg_import_flags & IMPORT_FLAGS_MASK_RAW);
if (r < 0)
return log_error_errno(r, "Failed to import image: %m");
@ -214,17 +264,32 @@ static int import_raw(int argc, char *argv[], void *userdata) {
static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n"
"Import container or virtual machine images.\n\n"
printf("%1$s [OPTIONS...] {COMMAND} ...\n"
"\n%4$sImport container or virtual machine images.%5$s\n"
"\n%2$sCommands:%3$s\n"
" tar FILE [NAME] Import a TAR image\n"
" raw FILE [NAME] Import a RAW image\n"
"\n%2$sOptions:%3$s\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n"
" --image-root=PATH Image root directory\n"
" --read-only Create a read-only image\n\n"
"Commands:\n"
" tar FILE [NAME] Import a TAR image\n"
" raw FILE [NAME] Import a RAW image\n",
program_invocation_short_name);
" --read-only Create a read-only image\n"
" --direct Import directly to specified file\n"
" --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n"
" instead of a directory\n"
" --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n"
" subvolume\n"
" --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n"
" regular disk images\n"
" --sync=BOOL Controls whether to sync() before completing\n"
" --offset=BYTES Offset to seek to in destination\n"
" --size-max=BYTES Maximum number of bytes to write to destination\n",
program_invocation_short_name,
ansi_underline(),
ansi_normal(),
ansi_highlight(),
ansi_normal());
return 0;
}
@ -236,6 +301,13 @@ static int parse_argv(int argc, char *argv[]) {
ARG_FORCE,
ARG_IMAGE_ROOT,
ARG_READ_ONLY,
ARG_DIRECT,
ARG_BTRFS_SUBVOL,
ARG_BTRFS_QUOTA,
ARG_CONVERT_QCOW2,
ARG_SYNC,
ARG_OFFSET,
ARG_SIZE_MAX,
};
static const struct option options[] = {
@ -244,10 +316,17 @@ static int parse_argv(int argc, char *argv[]) {
{ "force", no_argument, NULL, ARG_FORCE },
{ "image-root", required_argument, NULL, ARG_IMAGE_ROOT },
{ "read-only", no_argument, NULL, ARG_READ_ONLY },
{ "direct", no_argument, NULL, ARG_DIRECT },
{ "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL },
{ "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA },
{ "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 },
{ "sync", required_argument, NULL, ARG_SYNC },
{ "offset", required_argument, NULL, ARG_OFFSET },
{ "size-max", required_argument, NULL, ARG_SIZE_MAX },
{}
};
int c;
int r, c;
assert(argc >= 0);
assert(argv);
@ -274,6 +353,68 @@ static int parse_argv(int argc, char *argv[]) {
arg_import_flags |= IMPORT_READ_ONLY;
break;
case ARG_DIRECT:
arg_import_flags |= IMPORT_DIRECT;
break;
case ARG_BTRFS_SUBVOL:
r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
break;
case ARG_BTRFS_QUOTA:
r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
break;
case ARG_CONVERT_QCOW2:
r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r);
break;
case ARG_SYNC:
r = parse_boolean_argument("--sync=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
break;
case ARG_OFFSET: {
uint64_t u;
r = safe_atou64(optarg, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
arg_offset = u;
break;
}
case ARG_SIZE_MAX: {
uint64_t u;
r = parse_size(optarg, 1024, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
arg_size_max = u;
break;
}
case '?':
return -EINVAL;
@ -281,6 +422,15 @@ static int parse_argv(int argc, char *argv[]) {
assert_not_reached();
}
/* Make sure offset+size is still in the valid range if both set */
if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
((arg_size_max > (UINT64_MAX - arg_offset)) ||
!FILE_SIZE_VALID(arg_offset + arg_size_max)))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
return 1;
}
@ -295,6 +445,31 @@ static int import_main(int argc, char *argv[]) {
return dispatch_verb(argc, argv, verbs, NULL);
}
static void parse_env(void) {
int r;
/* Let's make these relatively low-level settings also controllable via env vars. User can then set
* them to systemd-import if they like to tweak behaviour */
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
r = getenv_bool("SYSTEMD_IMPORT_SYNC");
if (r >= 0)
SET_FLAG(arg_import_flags, IMPORT_SYNC, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
}
static int run(int argc, char *argv[]) {
int r;
@ -302,9 +477,11 @@ static int run(int argc, char *argv[]) {
log_parse_environment();
log_open();
parse_env();
r = parse_argv(argc, argv);
if (r <= 0)
return 0;
return r;
(void) ignore_signals(SIGPIPE);

220
src/import/pull-common.c
View File

@ -9,6 +9,7 @@
#include "dirent-util.h"
#include "escape.h"
#include "fd-util.h"
#include "hostname-util.h"
#include "io-util.h"
#include "memory-util.h"
#include "path-util.h"
@ -112,34 +113,6 @@ int pull_find_old_etags(
return 0;
}
int pull_make_local_copy(const char *final, const char *image_root, const char *local, PullFlags flags) {
const char *p;
int r;
assert(final);
assert(local);
if (!image_root)
image_root = "/var/lib/machines";
p = prefix_roota(image_root, local);
if (FLAGS_SET(flags, PULL_FORCE))
(void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = btrfs_subvol_snapshot(final, p,
BTRFS_SNAPSHOT_QUOTA|
BTRFS_SNAPSHOT_FALLBACK_COPY|
BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
BTRFS_SNAPSHOT_RECURSIVE);
if (r < 0)
return log_error_errno(r, "Failed to create local image: %m");
log_info("Created new local image '%s'.", local);
return 0;
}
static int hash_url(const char *url, char **ret) {
uint64_t h;
static const sd_id128_t k = SD_ID128_ARRAY(df,89,16,87,01,cc,42,30,98,ab,4a,19,a6,a5,63,4f);
@ -206,7 +179,9 @@ int pull_make_auxiliary_job(
const char *url,
int (*strip_suffixes)(const char *name, char **ret),
const char *suffix,
ImportVerify verify,
CurlGlue *glue,
PullJobOpenDisk on_open_disk,
PullJobFinished on_finished,
void *userdata) {
@ -238,45 +213,82 @@ int pull_make_auxiliary_job(
if (r < 0)
return r;
job->on_open_disk = on_open_disk;
job->on_finished = on_finished;
job->compressed_max = job->uncompressed_max = 1ULL * 1024ULL * 1024ULL;
job->calc_checksum = IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
*ret = TAKE_PTR(job);
return 0;
}
static bool is_checksum_file(const char *fn) {
/* Returns true if the specified filename refers to a checksum file we grok */
if (!fn)
return false;
return streq(fn, "SHA256SUMS") || endswith(fn, ".sha256");
}
static bool is_signature_file(const char *fn) {
/* Returns true if the specified filename refers to a signature file we grok (reminder:
* suse-style .sha256 files are inline signed) */
if (!fn)
return false;
return streq(fn, "SHA256SUMS.gpg") || endswith(fn, ".sha256");
}
int pull_make_verification_jobs(
PullJob **ret_checksum_job,
PullJob **ret_signature_job,
ImportVerify verify,
const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
const char *url,
CurlGlue *glue,
PullJobFinished on_finished,
void *userdata) {
_cleanup_(pull_job_unrefp) PullJob *checksum_job = NULL, *signature_job = NULL;
_cleanup_free_ char *fn = NULL;
int r;
assert(ret_checksum_job);
assert(ret_signature_job);
assert(verify >= 0);
assert(verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert(url);
assert(glue);
if (verify != IMPORT_VERIFY_NO) {
_cleanup_free_ char *checksum_url = NULL, *fn = NULL;
const char *chksums = NULL;
/* If verification is turned off, or if the checksum to validate is already specified we don't need
* to download a checksum file or signature, hence shortcut things */
if (verify == IMPORT_VERIFY_NO || checksum) {
*ret_checksum_job = *ret_signature_job = NULL;
return 0;
}
r = import_url_last_component(url, &fn);
if (r < 0 && r != -EADDRNOTAVAIL) /* EADDRNOTAVAIL means there was no last component, which is OK for
* us, we'll just assume it's not a checksum/signature file */
return r;
/* Acquire the checksum file if verification or signature verification is requested and the main file
* to acquire isn't a checksum or signature file anyway */
if (verify != IMPORT_VERIFY_NO && !is_checksum_file(fn) && !is_signature_file(fn)) {
_cleanup_free_ char *checksum_url = NULL;
const char *suffixed = NULL;
/* Queue jobs for the checksum file for the image. */
r = import_url_last_component(url, &fn);
if (r < 0)
return r;
chksums = strjoina(fn, ".sha256");
if (fn)
suffixed = strjoina(fn, ".sha256"); /* Start with the suse-style checksum (if there's a base filename) */
else
suffixed = "SHA256SUMS";
r = import_url_change_last_component(url, chksums, &checksum_url);
r = import_url_change_last_component(url, suffixed, &checksum_url);
if (r < 0)
return r;
@ -286,9 +298,10 @@ int pull_make_verification_jobs(
checksum_job->on_finished = on_finished;
checksum_job->uncompressed_max = checksum_job->compressed_max = 1ULL * 1024ULL * 1024ULL;
checksum_job->on_not_found = pull_job_restart_with_sha256sum; /* if this fails, look for ubuntu-style checksum */
}
if (verify == IMPORT_VERIFY_SIGNATURE) {
if (verify == IMPORT_VERIFY_SIGNATURE && !is_signature_file(fn)) {
_cleanup_free_ char *signature_url = NULL;
/* Queue job for the SHA256SUMS.gpg file for the image. */
@ -306,7 +319,6 @@ int pull_make_verification_jobs(
*ret_checksum_job = TAKE_PTR(checksum_job);
*ret_signature_job = TAKE_PTR(signature_job);
return 0;
}
@ -335,31 +347,33 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
r = import_url_last_component(job->url, &fn);
if (r < 0)
return log_oom();
return log_error_errno(r, "Failed to extract filename from URL '%s': %m", job->url);
if (!filename_is_valid(fn))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Cannot verify checksum, could not determine server-side file name.");
line = strjoina(job->checksum, " *", fn, "\n");
if (is_checksum_file(fn) || is_signature_file(fn)) /* We cannot verify checksum files or signature files with a checksum file */
return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
"Cannot verify checksum/signature files via themselves.");
line = strjoina(job->checksum, " *", fn, "\n"); /* string for binary mode */
p = memmem_safe(checksum_job->payload,
checksum_job->payload_size,
line,
strlen(line));
if (!p) {
line = strjoina(job->checksum, " ", fn, "\n");
line = strjoina(job->checksum, " ", fn, "\n"); /* string for text mode */
p = memmem_safe(checksum_job->payload,
checksum_job->payload_size,
line,
strlen(line));
}
/* Only counts if found at beginning of a line */
if (!p || (p != (char*) checksum_job->payload && p[-1] != '\n'))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"DOWNLOAD INVALID: Checksum of %s file did not checkout, file has been tampered with.", fn);
"DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", fn);
log_info("SHA256 checksum of %s is valid.", job->url);
return 1;
@ -472,10 +486,10 @@ static int verify_gpg(
pid = 0;
if (r < 0)
goto finish;
if (r != EXIT_SUCCESS) {
log_error("DOWNLOAD INVALID: Signature verification failed.");
r = -EBADMSG;
} else {
if (r != EXIT_SUCCESS)
r = log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"DOWNLOAD INVALID: Signature verification failed.");
else {
log_info("Signature verification succeeded.");
r = 0;
}
@ -491,6 +505,7 @@ finish:
}
int pull_verify(ImportVerify verify,
const char *checksum, /* Verify with literal checksum */
PullJob *main_job,
PullJob *checksum_job,
PullJob *signature_job,
@ -499,37 +514,79 @@ int pull_verify(ImportVerify verify,
PullJob *roothash_signature_job,
PullJob *verity_job) {
_cleanup_free_ char *fn = NULL;
VerificationStyle style;
PullJob *j;
PullJob *verify_job;
int r;
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert(main_job);
assert(main_job->state == PULL_JOB_DONE);
if (verify == IMPORT_VERIFY_NO)
if (verify == IMPORT_VERIFY_NO) /* verification turned off */
return 0;
assert(main_job->calc_checksum);
assert(main_job->checksum);
assert(checksum_job);
assert(checksum_job->state == PULL_JOB_DONE);
if (checksum) {
/* Verification by literal checksum */
assert(!checksum_job);
assert(!signature_job);
assert(!settings_job);
assert(!roothash_job);
assert(!roothash_signature_job);
assert(!verity_job);
if (!checksum_job->payload || checksum_job->payload_size <= 0)
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Checksum is empty, cannot verify.");
assert(main_job->calc_checksum);
assert(main_job->checksum);
FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
r = verify_one(checksum_job, j);
if (r < 0)
return r;
if (!strcaseeq(checksum, main_job->checksum))
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.",
main_job->url);
return 0;
}
if (verify == IMPORT_VERIFY_CHECKSUM)
r = import_url_last_component(main_job->url, &fn);
if (r < 0)
return log_error_errno(r, "Failed to extract filename from URL '%s': %m", main_job->url);
if (is_signature_file(fn))
return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
"Main download is a signature file, can't verify it.");
if (is_checksum_file(fn)) {
log_debug("Main download is a checksum file, can't validate its checksum with itself, skipping.");
verify_job = main_job;
} else {
PullJob *j;
assert(main_job->calc_checksum);
assert(main_job->checksum);
assert(checksum_job);
assert(checksum_job->state == PULL_JOB_DONE);
if (!checksum_job->payload || checksum_job->payload_size <= 0)
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Checksum is empty, cannot verify.");
FOREACH_POINTER(j, main_job, settings_job, roothash_job, roothash_signature_job, verity_job) {
r = verify_one(checksum_job, j);
if (r < 0)
return r;
}
verify_job = checksum_job;
}
if (verify != IMPORT_VERIFY_SIGNATURE)
return 0;
r = verification_style_from_url(checksum_job->url, &style);
assert(verify_job);
r = verification_style_from_url(verify_job->url, &style);
if (r < 0)
return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", checksum_job->url);
return log_error_errno(r, "Failed to determine verification style from URL '%s': %m", verify_job->url);
if (style == VERIFICATION_PER_DIRECTORY) {
assert(signature_job);
@ -539,9 +596,9 @@ int pull_verify(ImportVerify verify,
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"Signature is empty, cannot verify.");
return verify_gpg(checksum_job->payload, checksum_job->payload_size, signature_job->payload, signature_job->payload_size);
return verify_gpg(verify_job->payload, verify_job->payload_size, signature_job->payload, signature_job->payload_size);
} else
return verify_gpg(checksum_job->payload, checksum_job->payload_size, NULL, 0);
return verify_gpg(verify_job->payload, verify_job->payload_size, NULL, 0);
}
int verification_style_from_url(const char *url, VerificationStyle *ret) {
@ -595,3 +652,28 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) {
return 1;
}
bool pull_validate_local(const char *name, PullFlags flags) {
if (FLAGS_SET(flags, PULL_DIRECT))
return path_is_valid(name);
return hostname_is_valid(name, 0);
}
int pull_url_needs_checksum(const char *url) {
_cleanup_free_ char *fn = NULL;
int r;
/* Returns true if we need to validate this resource via a hash value. This returns true for all
* files — except for gpg signature files and SHA256SUMS files and the like, which are validated with
* a validation tool like gpg. */
r = import_url_last_component(url, &fn);
if (r == -EADDRNOTAVAIL) /* no last component? then let's assume it's not a signature/checksum file */
return false;
if (r < 0)
return r;
return !is_checksum_file(fn) && !is_signature_file(fn);
}

32
src/import/pull-common.h
View File

@ -7,27 +7,31 @@
#include "pull-job.h"
typedef enum PullFlags {
PULL_FORCE = 1 << 0, /* replace existing image */
PULL_SETTINGS = 1 << 1, /* .nspawn settings file */
PULL_ROOTHASH = 1 << 2, /* only for raw: .roothash file for verity */
PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: .roothash.p7s file for verity */
PULL_VERITY = 1 << 4, /* only for raw: .verity file for verity */
PULL_FORCE = 1 << 0, /* replace existing image */
PULL_READ_ONLY = 1 << 1, /* make generated image read-only */
PULL_SETTINGS = 1 << 1, /* download .nspawn settings file */
PULL_ROOTHASH = 1 << 2, /* only for raw: download .roothash file for verity */
PULL_ROOTHASH_SIGNATURE = 1 << 3, /* only for raw: download .roothash.p7s file for verity */
PULL_VERITY = 1 << 4, /* only for raw: download .verity file for verity */
PULL_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */
PULL_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */
PULL_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */
PULL_DIRECT = 1 << 5, /* download without rename games */
PULL_SYNC = 1 << 6, /* fsync() right before we are done */
/* The supported flags for the tar and the raw pulling */
PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_SETTINGS,
PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY,
PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC,
PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC,
} PullFlags;
int pull_make_local_copy(const char *final, const char *root, const char *local, PullFlags flags);
int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags);
int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata);
int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
int pull_verify(ImportVerify verify, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
typedef enum VerificationStyle {
VERIFICATION_PER_FILE, /* SuSE-style ".sha256" files with inline gpg signature */
@ -39,3 +43,7 @@ typedef enum VerificationStyle {
int verification_style_from_url(const char *url, VerificationStyle *style);
int pull_job_restart_with_sha256sum(PullJob *job, char **ret);
bool pull_validate_local(const char *name, PullFlags flags);
int pull_url_needs_checksum(const char *url);

183
src/import/pull-job.c
View File

@ -7,6 +7,7 @@
#include "alloc-util.h"
#include "fd-util.h"
#include "format-util.h"
#include "fs-util.h"
#include "gcrypt-util.h"
#include "hexdecoct.h"
#include "import-util.h"
@ -19,15 +20,25 @@
#include "strv.h"
#include "xattr-util.h"
void pull_job_close_disk_fd(PullJob *j) {
if (!j)
return;
if (j->close_disk_fd)
safe_close(j->disk_fd);
j->disk_fd = -1;
}
PullJob* pull_job_unref(PullJob *j) {
if (!j)
return NULL;
pull_job_close_disk_fd(j);
curl_glue_remove_and_free(j->glue, j->curl);
curl_slist_free_all(j->request_header);
safe_close(j->disk_fd);
import_compress_free(&j->compress);
if (j->checksum_context)
@ -116,15 +127,13 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
return;
if (result != CURLE_OK) {
log_error("Transfer failed: %s", curl_easy_strerror(result));
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Transfer failed: %s", curl_easy_strerror(result));
goto finish;
}
code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto finish;
} else if (status == 304) {
log_info("Image already downloaded. Skipping download.");
@ -150,8 +159,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto finish;
}
@ -160,25 +168,23 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
}
}
log_error("HTTP request to %s failed with code %li.", j->url, status);
r = -EIO;
r = log_error_errno(
status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */
"HTTP request to %s failed with code %li.", j->url, status);
goto finish;
} else if (status < 200) {
log_error("HTTP request to %s finished with unexpected code %li.", j->url, status);
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status);
goto finish;
}
if (j->state != PULL_JOB_RUNNING) {
log_error("Premature connection termination.");
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.");
goto finish;
}
if (j->content_length != UINT64_MAX &&
j->content_length != j->written_compressed) {
log_error("Download truncated.");
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Download truncated.");
goto finish;
}
@ -187,8 +193,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
k = gcry_md_read(j->checksum_context, GCRY_MD_SHA256);
if (!k) {
log_error("Failed to get checksum.");
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
goto finish;
}
@ -201,28 +206,57 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
log_debug("SHA256 of %s is %s.", j->url, j->checksum);
}
if (j->disk_fd >= 0 && j->allow_sparse) {
/* Make sure the file size is right, in case the file was
* sparse and we just seeked for the last part */
/* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no
* offset is specified, which indicates we only own the file partially) */
if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
r = log_error_errno(errno, "Failed to truncate file: %m");
goto finish;
}
if (j->disk_fd >= 0) {
if (j->etag)
(void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
if (j->url)
(void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
if (S_ISREG(j->disk_stat.st_mode)) {
if (j->mtime != 0) {
struct timespec ut[2];
if (j->offset == UINT64_MAX) {
timespec_store(&ut[0], j->mtime);
ut[1] = ut[0];
(void) futimens(j->disk_fd, ut);
if (j->written_compressed > 0) {
/* Make sure the file size is right, in case the file was sparse and we just seeked
* for the last part */
if (ftruncate(j->disk_fd, j->written_uncompressed) < 0) {
r = log_error_errno(errno, "Failed to truncate file: %m");
goto finish;
}
}
(void) fd_setcrtime(j->disk_fd, j->mtime);
if (j->etag)
(void) fsetxattr(j->disk_fd, "user.source_etag", j->etag, strlen(j->etag), 0);
if (j->url)
(void) fsetxattr(j->disk_fd, "user.source_url", j->url, strlen(j->url), 0);
if (j->mtime != 0) {
struct timespec ut;
timespec_store(&ut, j->mtime);
if (futimens(j->disk_fd, (struct timespec[]) { ut, ut }) < 0)
log_debug_errno(errno, "Failed to adjust atime/mtime of created image, ignoring: %m");
r = fd_setcrtime(j->disk_fd, j->mtime);
if (r < 0)
log_debug_errno(r, "Failed to adjust crtime of created image, ignoring: %m");
}
}
if (j->sync) {
r = fsync_full(j->disk_fd);
if (r < 0) {
log_error_errno(r, "Failed to synchronize file to disk: %m");
goto finish;
}
}
} else if (S_ISBLK(j->disk_stat.st_mode) && j->sync) {
if (fsync(j->disk_fd) < 0) {
r = log_error_errno(errno, "Failed to synchronize block device: %m");
goto finish;
}
}
}
@ -234,37 +268,46 @@ finish:
static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) {
PullJob *j = userdata;
ssize_t n;
bool too_much = false;
int r;
assert(j);
assert(p);
assert(sz > 0);
if (sz <= 0)
return 0;
if (j->written_uncompressed > UINT64_MAX - sz)
return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow");
if (j->written_uncompressed + sz < j->written_uncompressed)
return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW),
"File too large, overflow");
if (j->written_uncompressed >= j->uncompressed_max) {
too_much = true;
goto finish;
}
if (j->written_uncompressed + sz > j->uncompressed_max)
return log_error_errno(SYNTHETIC_ERRNO(EFBIG),
"File overly large, refusing");
if (j->written_uncompressed + sz > j->uncompressed_max) {
too_much = true;
sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory
* already, we might as well write it to
* disk to the max */
}
if (j->disk_fd >= 0) {
if (j->allow_sparse)
n = sparse_write(j->disk_fd, p, sz, 64);
else {
n = write(j->disk_fd, p, sz);
if (n < 0)
n = -errno;
}
if (n < 0)
return log_error_errno((int) n, "Failed to write file: %m");
if ((size_t) n < sz)
return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
} else {
if (S_ISREG(j->disk_stat.st_mode)) {
ssize_t n;
n = sparse_write(j->disk_fd, p, sz, 64);
if (n < 0)
return log_error_errno((int) n, "Failed to write file: %m");
if ((size_t) n < sz)
return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write");
} else {
r = loop_write(j->disk_fd, p, sz, false);
if (r < 0)
return log_error_errno(r, "Failed to write file: %m");
}
}
if (j->disk_fd < 0 || j->force_memory) {
if (!GREEDY_REALLOC(j->payload, j->payload_size + sz))
return log_oom();
@ -274,6 +317,10 @@ static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata)
j->written_uncompressed += sz;
finish:
if (too_much)
return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing.");
return 0;
}
@ -321,15 +368,12 @@ static int pull_job_open_disk(PullJob *j) {
}
if (j->disk_fd >= 0) {
/* Check if we can do sparse files */
if (fstat(j->disk_fd, &j->disk_stat) < 0)
return log_error_errno(errno, "Failed to stat disk file: %m");
if (lseek(j->disk_fd, SEEK_SET, 0) == 0)
j->allow_sparse = true;
else {
if (errno != ESPIPE)
if (j->offset != UINT64_MAX) {
if (lseek(j->disk_fd, j->offset, SEEK_SET) == (off_t) -1)
return log_error_errno(errno, "Failed to seek on file descriptor: %m");
j->allow_sparse = false;
}
}
@ -461,8 +505,7 @@ static size_t pull_job_header_callback(void *contents, size_t size, size_t nmemb
code = curl_easy_getinfo(j->curl, CURLINFO_RESPONSE_CODE, &status);
if (code != CURLE_OK) {
log_error("Failed to retrieve response code: %s", curl_easy_strerror(code));
r = -EIO;
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to retrieve response code: %s", curl_easy_strerror(code));
goto fail;
}
@ -576,7 +619,12 @@ static int pull_job_progress_callback(void *userdata, curl_off_t dltotal, curl_o
return 0;
}
int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata) {
int pull_job_new(
PullJob **ret,
const char *url,
CurlGlue *glue,
void *userdata) {
_cleanup_(pull_job_unrefp) PullJob *j = NULL;
_cleanup_free_ char *u = NULL;
@ -595,6 +643,7 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata)
*j = (PullJob) {
.state = PULL_JOB_INIT,
.disk_fd = -1,
.close_disk_fd = true,
.userdata = userdata,
.glue = glue,
.content_length = UINT64_MAX,
@ -602,6 +651,8 @@ int pull_job_new(PullJob **ret, const char *url, CurlGlue *glue, void *userdata)
.compressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
.uncompressed_max = 64LLU * 1024LLU * 1024LLU * 1024LLU, /* 64GB safety limit */
.url = TAKE_PTR(u),
.offset = UINT64_MAX,
.sync = true,
};
*ret = TAKE_PTR(j);

11
src/import/pull-job.h
View File

@ -2,10 +2,12 @@
#pragma once
#include <gcrypt.h>
#include <sys/stat.h>
#include "curl-util.h"
#include "import-compress.h"
#include "macro.h"
#include "pull-common.h"
typedef struct PullJob PullJob;
@ -51,6 +53,7 @@ struct PullJob {
uint64_t content_length;
uint64_t written_compressed;
uint64_t written_uncompressed;
uint64_t offset;
uint64_t uncompressed_max;
uint64_t compressed_max;
@ -59,6 +62,8 @@ struct PullJob {
size_t payload_size;
int disk_fd;
bool close_disk_fd;
struct stat disk_stat;
usec_t mtime;
@ -68,12 +73,12 @@ struct PullJob {
usec_t start_usec;
usec_t last_status_usec;
bool allow_sparse;
bool calc_checksum;
gcry_md_hd_t checksum_context;
char *checksum;
bool sync;
bool force_memory;
};
int pull_job_new(PullJob **job, const char *url, CurlGlue *glue, void *userdata);
@ -83,4 +88,6 @@ int pull_job_begin(PullJob *j);
void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result);
void pull_job_close_disk_fd(PullJob *j);
DEFINE_TRIVIAL_CLEANUP_FUNC(PullJob*, pull_job_unref);

491
src/import/pull-raw.c
View File

@ -15,6 +15,7 @@
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
#include "install-file.h"
#include "macro.h"
#include "mkdir.h"
#include "path-util.h"
@ -46,6 +47,8 @@ struct RawPull {
ImportVerify verify;
char *image_root;
uint64_t offset;
PullJob *raw_job;
PullJob *checksum_job;
PullJob *signature_job;
@ -57,7 +60,8 @@ struct RawPull {
RawPullFinished on_finished;
void *userdata;
char *local;
char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the
* machine name of the final copy we make */
char *final_path;
char *temp_path;
@ -73,6 +77,8 @@ struct RawPull {
char *verity_path;
char *verity_temp_path;
char *checksum;
};
RawPull* raw_pull_unref(RawPull *i) {
@ -103,6 +109,7 @@ RawPull* raw_pull_unref(RawPull *i) {
free(i->verity_path);
free(i->image_root);
free(i->local);
free(i->checksum);
return mfree(i);
}
@ -148,6 +155,7 @@ int raw_pull_new(
.image_root = TAKE_PTR(root),
.event = TAKE_PTR(e),
.glue = TAKE_PTR(g),
.offset = UINT64_MAX,
};
i->glue->on_finished = pull_job_curl_on_finished;
@ -230,12 +238,20 @@ static void raw_pull_report_progress(RawPull *i, RawProgress p) {
}
static int raw_pull_maybe_convert_qcow2(RawPull *i) {
_cleanup_(unlink_and_freep) char *t = NULL;
_cleanup_close_ int converted_fd = -1;
_cleanup_free_ char *t = NULL;
_cleanup_free_ char *f = NULL;
int r;
assert(i);
assert(i->raw_job);
assert(!FLAGS_SET(i->flags, PULL_DIRECT));
if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2))
return 0;
assert(i->final_path);
assert(i->raw_job->close_disk_fd);
r = qcow2_detect(i->raw_job->disk_fd);
if (r < 0)
@ -244,32 +260,35 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) {
return 0;
/* This is a QCOW2 image, let's convert it */
r = tempfn_random(i->final_path, NULL, &t);
r = tempfn_random(i->final_path, NULL, &f);
if (r < 0)
return log_oom();
converted_fd = open(t, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
converted_fd = open(f, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (converted_fd < 0)
return log_error_errno(errno, "Failed to create %s: %m", t);
return log_error_errno(errno, "Failed to create %s: %m", f);
t = TAKE_PTR(f);
(void) import_set_nocow_and_log(converted_fd, t);
log_info("Unpacking QCOW2 file.");
r = qcow2_convert(i->raw_job->disk_fd, converted_fd);
if (r < 0) {
(void) unlink(t);
if (r < 0)
return log_error_errno(r, "Failed to convert qcow2 image: %m");
}
(void) unlink(i->temp_path);
free_and_replace(i->temp_path, t);
unlink_and_free(i->temp_path);
i->temp_path = TAKE_PTR(t);
CLOSE_AND_REPLACE(i->raw_job->disk_fd, converted_fd);
return 1;
}
static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field) {
static int raw_pull_determine_path(
RawPull *i,
const char *suffix,
char **field /* input + output (!) */) {
int r;
assert(i);
@ -290,7 +309,7 @@ static int raw_pull_determine_path(RawPull *i, const char *suffix, char **field)
static int raw_pull_copy_auxiliary_file(
RawPull *i,
const char *suffix,
char **path) {
char **path /* input + output (!) */) {
const char *local;
int r;
@ -305,7 +324,14 @@ static int raw_pull_copy_auxiliary_file(
local = strjoina(i->image_root, "/", i->local, suffix);
r = copy_file_atomic(*path, local, 0644, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
r = copy_file_atomic(
*path,
local,
0644,
0, 0,
COPY_REFLINK |
(FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
(FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
if (r == -EEXIST)
log_warning_errno(r, "File %s already exists, not replacing.", local);
else if (r == -ENOENT)
@ -319,13 +345,15 @@ static int raw_pull_copy_auxiliary_file(
}
static int raw_pull_make_local_copy(RawPull *i) {
_cleanup_free_ char *tp = NULL;
_cleanup_(unlink_and_freep) char *tp = NULL;
_cleanup_free_ char *f = NULL;
_cleanup_close_ int dfd = -1;
const char *p;
int r;
assert(i);
assert(i->raw_job);
assert(!FLAGS_SET(i->flags, PULL_DIRECT));
if (!i->local)
return 0;
@ -342,6 +370,7 @@ static int raw_pull_make_local_copy(RawPull *i) {
/* We freshly downloaded the image, use it */
assert(i->raw_job->disk_fd >= 0);
assert(i->offset == UINT64_MAX);
if (lseek(i->raw_job->disk_fd, SEEK_SET, 0) == (off_t) -1)
return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m");
@ -349,38 +378,38 @@ static int raw_pull_make_local_copy(RawPull *i) {
p = strjoina(i->image_root, "/", i->local, ".raw");
if (FLAGS_SET(i->flags, PULL_FORCE))
(void) rm_rf(p, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
r = tempfn_random(p, NULL, &tp);
r = tempfn_random(p, NULL, &f);
if (r < 0)
return log_oom();
dfd = open(tp, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (dfd < 0)
return log_error_errno(errno, "Failed to create writable copy of image: %m");
tp = TAKE_PTR(f);
/* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs,
* since it reduces fragmentation caused by not allowing in-place writes. */
(void) import_set_nocow_and_log(dfd, tp);
r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK);
if (r < 0) {
(void) unlink(tp);
if (r < 0)
return log_error_errno(r, "Failed to make writable copy of image: %m");
}
(void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME);
(void) copy_xattr(i->raw_job->disk_fd, dfd, 0);
dfd = safe_close(dfd);
r = rename(tp, p);
if (r < 0) {
r = log_error_errno(errno, "Failed to move writable image into place: %m");
(void) unlink(tp);
return r;
}
r = install_file(AT_FDCWD, tp,
AT_FDCWD, p,
(i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
(i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0)
return log_error_errno(errno, "Failed to move local image into place '%s': %m", p);
tp = mfree(tp);
log_info("Created new local image '%s'.", i->local);
@ -442,9 +471,10 @@ static int raw_pull_rename_auxiliary_file(
int r;
assert(i);
assert(temp_path);
assert(suffix);
assert(path);
assert(temp_path);
assert(*temp_path);
assert(suffix);
/* Regenerate final name for this auxiliary file, we might know the etag of the file now, and we should
* incorporate it in the file name if we can */
@ -453,61 +483,78 @@ static int raw_pull_rename_auxiliary_file(
if (r < 0)
return r;
r = import_make_read_only(*temp_path);
r = install_file(
AT_FDCWD, *temp_path,
AT_FDCWD, *path,
INSTALL_READ_ONLY|
(i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0)
return r;
r = rename_noreplace(AT_FDCWD, *temp_path, AT_FDCWD, *path);
if (r < 0)
return log_error_errno(r, "Failed to rename file %s to %s: %m", *temp_path, *path);
return log_error_errno(r, "Failed to move '%s' into place: %m", *path);
*temp_path = mfree(*temp_path);
return 1;
}
static void raw_pull_job_on_finished(PullJob *j) {
RawPull *i;
PullJob *jj;
int r;
assert(j);
assert(j->userdata);
i = j->userdata;
if (j == i->settings_job) {
if (j->error != 0)
log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
} else if (j == i->roothash_job) {
if (j->error != 0)
log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
} else if (j == i->roothash_signature_job) {
if (j->error != 0)
log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without.");
} else if (j == i->verity_job) {
if (j->error != 0)
log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without. %s", j->url);
} else if (j->error != 0 && j != i->signature_job) {
if (j == i->checksum_job)
log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
else
log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
r = j->error;
goto finish;
if (j->error != 0) {
/* Only the main job and the checksum job are fatal if they fail. The other fails are just
* "decoration", that we'll download if we can. The signature job isn't fatal here because we
* might not actually need it in case Suse style signatures are used, that are inline in the
* checksum file. */
if (j == i->raw_job) {
if (j->error == ENOMEDIUM) /* HTTP 404 */
r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
else
r = log_error_errno(j->error, "Failed to retrieve image file.");
goto finish;
} else if (j == i->checksum_job) {
r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
goto finish;
} else if (j == i->signature_job)
log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
else if (j == i->settings_job)
log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
else if (j == i->roothash_job)
log_info_errno(j->error, "Root hash file could not be retrieved, proceeding without.");
else if (j == i->roothash_signature_job)
log_info_errno(j->error, "Root hash signature file could not be retrieved, proceeding without.");
else if (j == i->verity_job)
log_info_errno(j->error, "Verity integrity file could not be retrieved, proceeding without.");
else
assert_not_reached();
}
/* This is invoked if either the download completed successfully, or the download was skipped because
* we already have the etag. In this case ->etag_exists is true.
*
* We only do something when we got all three files */
* We only do something when we got all files */
if (!raw_pull_is_done(i))
return;
if (i->signature_job && i->signature_job->error != 0) {
VerificationStyle style;
PullJob *verify_job;
r = verification_style_from_url(i->checksum_job->url, &style);
/* The signature job failed. Let's see if we actually need it */
verify_job = i->checksum_job ?: i->raw_job; /* if the checksum job doesn't exist this must be
* because the main job is the checksum file
* itself */
assert(verify_job);
r = verification_style_from_url(verify_job->url, &style);
if (r < 0) {
log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
goto finish;
@ -517,32 +564,21 @@ static void raw_pull_job_on_finished(PullJob *j) {
* in per-directory verification mode, since only
* then the signature is detached, and thus a file
* of its own. */
log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
r = i->signature_job->error;
r = log_error_errno(i->signature_job->error,
"Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
goto finish;
}
}
if (i->settings_job)
i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
if (i->roothash_job)
i->roothash_job->disk_fd = safe_close(i->roothash_job->disk_fd);
if (i->roothash_signature_job)
i->roothash_signature_job->disk_fd = safe_close(i->roothash_signature_job->disk_fd);
if (i->verity_job)
i->verity_job->disk_fd = safe_close(i->verity_job->disk_fd);
r = raw_pull_determine_path(i, ".raw", &i->final_path);
if (r < 0)
goto finish;
/* Let's close these auxiliary files now, we don't need access to them anymore. */
FOREACH_POINTER(jj, i->settings_job, i->roothash_job, i->roothash_signature_job, i->verity_job)
pull_job_close_disk_fd(jj);
if (!i->raw_job->etag_exists) {
/* This is a new download, verify it, and move it into place */
assert(i->raw_job->disk_fd >= 0);
raw_pull_report_progress(i, RAW_VERIFYING);
r = pull_verify(i->verify,
i->checksum,
i->raw_job,
i->checksum_job,
i->signature_job,
@ -552,51 +588,92 @@ static void raw_pull_job_on_finished(PullJob *j) {
i->verity_job);
if (r < 0)
goto finish;
}
raw_pull_report_progress(i, RAW_UNPACKING);
r = raw_pull_maybe_convert_qcow2(i);
if (r < 0)
goto finish;
if (i->flags & PULL_DIRECT) {
assert(!i->settings_job);
assert(!i->roothash_job);
assert(!i->roothash_signature_job);
assert(!i->verity_job);
raw_pull_report_progress(i, RAW_FINALIZING);
if (i->raw_job->etag) {
/* Only make a read-only copy if ETag header is set. */
r = import_make_read_only_fd(i->raw_job->disk_fd);
if (r < 0)
goto finish;
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
if (i->local) {
r = install_file(AT_FDCWD, i->local,
AT_FDCWD, NULL,
((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) |
(i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0) {
log_error_errno(r, "Failed to rename raw file to %s: %m", i->final_path);
log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local);
goto finish;
}
}
} else {
r = raw_pull_determine_path(i, ".raw", &i->final_path);
if (r < 0)
goto finish;
i->temp_path = mfree(i->temp_path);
if (!i->raw_job->etag_exists) {
/* This is a new download, verify it, and move it into place */
if (i->roothash_job &&
i->roothash_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
assert(i->temp_path);
assert(i->final_path);
raw_pull_report_progress(i, RAW_UNPACKING);
r = raw_pull_maybe_convert_qcow2(i);
if (r < 0)
goto finish;
raw_pull_report_progress(i, RAW_FINALIZING);
r = install_file(AT_FDCWD, i->temp_path,
AT_FDCWD, i->final_path,
INSTALL_READ_ONLY|
(i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0) {
log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path);
goto finish;
}
i->temp_path = mfree(i->temp_path);
if (i->settings_job &&
i->settings_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
if (r < 0)
goto finish;
}
if (i->roothash_job &&
i->roothash_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".roothash", &i->roothash_temp_path, &i->roothash_path);
if (r < 0)
goto finish;
}
if (i->roothash_signature_job &&
i->roothash_signature_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_temp_path, &i->roothash_signature_path);
if (r < 0)
goto finish;
}
if (i->verity_job &&
i->verity_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".verity", &i->verity_temp_path, &i->verity_path);
if (r < 0)
goto finish;
}
}
if (i->settings_job &&
i->settings_job->error == 0) {
r = raw_pull_rename_auxiliary_file(i, ".nspawn", &i->settings_temp_path, &i->settings_path);
if (r < 0)
goto finish;
}
raw_pull_report_progress(i, RAW_COPYING);
r = raw_pull_make_local_copy(i);
if (r < 0)
goto finish;
}
raw_pull_report_progress(i, RAW_COPYING);
r = raw_pull_make_local_copy(i);
if (r < 0)
goto finish;
r = 0;
finish:
@ -610,7 +687,7 @@ static int raw_pull_job_on_open_disk_generic(
RawPull *i,
PullJob *j,
const char *extra,
char **temp_path) {
char **temp_path /* input + output */) {
int r;
@ -619,6 +696,8 @@ static int raw_pull_job_on_open_disk_generic(
assert(extra);
assert(temp_path);
assert(!FLAGS_SET(i->flags, PULL_DIRECT));
if (!*temp_path) {
r = tempfn_random_child(i->image_root, extra, temp_path);
if (r < 0)
@ -643,12 +722,34 @@ static int raw_pull_job_on_open_disk_raw(PullJob *j) {
i = j->userdata;
assert(i->raw_job == j);
assert(j->disk_fd < 0);
r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
if (r < 0)
return r;
if (i->flags & PULL_DIRECT) {
if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */
j->disk_fd = STDOUT_FILENO;
j->close_disk_fd = false;
return 0;
}
(void) mkdir_parents_label(i->local, 0700);
j->disk_fd = open(i->local, O_RDWR|O_NOCTTY|O_CLOEXEC|(i->offset == UINT64_MAX ? O_TRUNC|O_CREAT : 0), 0664);
if (j->disk_fd < 0)
return log_error_errno(errno, "Failed to open destination '%s': %m", i->local);
if (i->offset == UINT64_MAX)
(void) import_set_nocow_and_log(j->disk_fd, i->local);
} else {
r = raw_pull_job_on_open_disk_generic(i, j, "raw", &i->temp_path);
if (r < 0)
return r;
assert(i->offset == UINT64_MAX);
(void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
}
(void) import_set_nocow_and_log(j->disk_fd, i->temp_path);
return 0;
}
@ -715,20 +816,29 @@ int raw_pull_start(
RawPull *i,
const char *url,
const char *local,
uint64_t offset,
uint64_t size_max,
PullFlags flags,
ImportVerify verify) {
ImportVerify verify,
const char *checksum) {
PullJob *j;
int r;
assert(i);
assert(verify < _IMPORT_VERIFY_MAX);
assert(verify >= 0);
assert(url);
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert(!(flags & ~PULL_FLAGS_MASK_RAW));
assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT));
assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT));
assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum);
if (!http_url_is_valid(url))
return -EINVAL;
if (local && !hostname_is_valid(local, 0))
if (local && !pull_validate_local(local, flags))
return -EINVAL;
if (i->raw_job)
@ -738,6 +848,10 @@ int raw_pull_start(
if (r < 0)
return r;
r = free_and_strdup(&i->checksum, checksum);
if (r < 0)
return r;
i->flags = flags;
i->verify = verify;
@ -748,98 +862,121 @@ int raw_pull_start(
i->raw_job->on_finished = raw_pull_job_on_finished;
i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
i->raw_job->on_progress = raw_pull_job_on_progress;
i->raw_job->calc_checksum = verify != IMPORT_VERIFY_NO;
r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
if (r < 0)
return r;
if (checksum)
i->raw_job->calc_checksum = true;
else if (verify != IMPORT_VERIFY_NO) {
/* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its
* signature, which we let gpg verify instead. */
r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, raw_pull_job_on_finished, i);
r = pull_url_needs_checksum(url);
if (r < 0)
return r;
i->raw_job->calc_checksum = r;
i->raw_job->force_memory = true; /* make sure this is both written to disk if that's
* requested and into memory, since we need to verify it */
}
if (size_max != UINT64_MAX)
i->raw_job->uncompressed_max = size_max;
if (offset != UINT64_MAX)
i->raw_job->offset = i->offset = offset;
if (!FLAGS_SET(flags, PULL_DIRECT)) {
r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags);
if (r < 0)
return r;
}
r = pull_make_verification_jobs(
&i->checksum_job,
&i->signature_job,
verify,
i->checksum,
url,
i->glue,
raw_pull_job_on_finished,
i);
if (r < 0)
return r;
if (FLAGS_SET(flags, PULL_SETTINGS)) {
r = pull_make_auxiliary_job(&i->settings_job, url, raw_strip_suffixes, ".nspawn", i->glue, raw_pull_job_on_finished, i);
r = pull_make_auxiliary_job(
&i->settings_job,
url,
raw_strip_suffixes,
".nspawn",
verify,
i->glue,
raw_pull_job_on_open_disk_settings,
raw_pull_job_on_finished,
i);
if (r < 0)
return r;
i->settings_job->on_open_disk = raw_pull_job_on_open_disk_settings;
i->settings_job->on_progress = raw_pull_job_on_progress;
i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
if (FLAGS_SET(flags, PULL_ROOTHASH)) {
r = pull_make_auxiliary_job(&i->roothash_job, url, raw_strip_suffixes, ".roothash", i->glue, raw_pull_job_on_finished, i);
r = pull_make_auxiliary_job(
&i->roothash_job,
url,
raw_strip_suffixes,
".roothash",
verify,
i->glue,
raw_pull_job_on_open_disk_roothash,
raw_pull_job_on_finished,
i);
if (r < 0)
return r;
i->roothash_job->on_open_disk = raw_pull_job_on_open_disk_roothash;
i->roothash_job->on_progress = raw_pull_job_on_progress;
i->roothash_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) {
r = pull_make_auxiliary_job(&i->roothash_signature_job, url, raw_strip_suffixes, ".roothash.p7s", i->glue, raw_pull_job_on_finished, i);
r = pull_make_auxiliary_job(
&i->roothash_signature_job,
url,
raw_strip_suffixes,
".roothash.p7s",
verify,
i->glue,
raw_pull_job_on_open_disk_roothash_signature,
raw_pull_job_on_finished,
i);
if (r < 0)
return r;
i->roothash_signature_job->on_open_disk = raw_pull_job_on_open_disk_roothash_signature;
i->roothash_signature_job->on_progress = raw_pull_job_on_progress;
i->roothash_signature_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
if (FLAGS_SET(flags, PULL_VERITY)) {
r = pull_make_auxiliary_job(&i->verity_job, url, raw_strip_suffixes, ".verity", i->glue, raw_pull_job_on_finished, i);
if (r < 0)
return r;
i->verity_job->on_open_disk = raw_pull_job_on_open_disk_verity;
i->verity_job->on_progress = raw_pull_job_on_progress;
i->verity_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
r = pull_job_begin(i->raw_job);
if (r < 0)
return r;
if (i->checksum_job) {
i->checksum_job->on_progress = raw_pull_job_on_progress;
i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
r = pull_job_begin(i->checksum_job);
r = pull_make_auxiliary_job(
&i->verity_job,
url,
raw_strip_suffixes,
".verity",
verify,
i->glue,
raw_pull_job_on_open_disk_verity,
raw_pull_job_on_finished,
i);
if (r < 0)
return r;
}
if (i->signature_job) {
i->signature_job->on_progress = raw_pull_job_on_progress;
FOREACH_POINTER(j,
i->raw_job,
i->checksum_job,
i->signature_job,
i->settings_job,
i->roothash_job,
i->roothash_signature_job,
i->verity_job) {
r = pull_job_begin(i->signature_job);
if (r < 0)
return r;
}
if (!j)
continue;
if (i->settings_job) {
r = pull_job_begin(i->settings_job);
if (r < 0)
return r;
}
j->on_progress = raw_pull_job_on_progress;
j->sync = FLAGS_SET(flags, PULL_SYNC);
if (i->roothash_job) {
r = pull_job_begin(i->roothash_job);
if (r < 0)
return r;
}
if (i->roothash_signature_job) {
r = pull_job_begin(i->roothash_signature_job);
if (r < 0)
return r;
}
if (i->verity_job) {
r = pull_job_begin(i->verity_job);
r = pull_job_begin(j);
if (r < 0)
return r;
}

2
src/import/pull-raw.h
View File

@ -16,4 +16,4 @@ RawPull* raw_pull_unref(RawPull *pull);
DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
int raw_pull_start(RawPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum);

314
src/import/pull-tar.c
View File

@ -14,6 +14,7 @@
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
#include "install-file.h"
#include "macro.h"
#include "mkdir.h"
#include "path-util.h"
@ -25,6 +26,7 @@
#include "string-util.h"
#include "strv.h"
#include "tmpfile-util.h"
#include "user-util.h"
#include "utf8.h"
#include "util.h"
#include "web-util.h"
@ -61,6 +63,8 @@ struct TarPull {
char *settings_path;
char *settings_temp_path;
char *checksum;
};
TarPull* tar_pull_unref(TarPull *i) {
@ -87,6 +91,7 @@ TarPull* tar_pull_unref(TarPull *i) {
free(i->settings_path);
free(i->image_root);
free(i->local);
free(i->checksum);
return mfree(i);
}
@ -194,7 +199,10 @@ static void tar_pull_report_progress(TarPull *i, TarProgress p) {
log_debug("Combined progress %u%%", percent);
}
static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field) {
static int tar_pull_determine_path(
TarPull *i,
const char *suffix,
char **field /* input + output (!) */) {
int r;
assert(i);
@ -213,6 +221,8 @@ static int tar_pull_determine_path(TarPull *i, const char *suffix, char **field)
}
static int tar_pull_make_local_copy(TarPull *i) {
_cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
const char *p;
int r;
assert(i);
@ -221,9 +231,38 @@ static int tar_pull_make_local_copy(TarPull *i) {
if (!i->local)
return 0;
r = pull_make_local_copy(i->final_path, i->image_root, i->local, i->flags);
assert(i->final_path);
p = prefix_roota(i->image_root, i->local);
r = tempfn_random(p, NULL, &t);
if (r < 0)
return r;
return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
if (i->flags & PULL_BTRFS_SUBVOL)
r = btrfs_subvol_snapshot(
i->final_path,
t,
(i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
BTRFS_SNAPSHOT_FALLBACK_COPY|
BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
BTRFS_SNAPSHOT_RECURSIVE);
else
r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS);
if (r < 0)
return log_error_errno(r, "Failed to create local image: %m");
r = install_file(AT_FDCWD, t,
AT_FDCWD, p,
(i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
(i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
(i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0)
return log_error_errno(r, "Failed to install local image '%s': %m", p);
t = mfree(t);
log_info("Created new local image '%s'.", i->local);
if (FLAGS_SET(i->flags, PULL_SETTINGS)) {
const char *local_settings;
@ -235,7 +274,14 @@ static int tar_pull_make_local_copy(TarPull *i) {
local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
r = copy_file_atomic(i->settings_path, local_settings, 0664, 0, 0, COPY_REFLINK | (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0));
r = copy_file_atomic(
i->settings_path,
local_settings,
0664,
0, 0,
COPY_REFLINK |
(FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
(FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
if (r == -EEXIST)
log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
else if (r == -ENOENT)
@ -274,17 +320,22 @@ static void tar_pull_job_on_finished(PullJob *j) {
i = j->userdata;
if (j == i->settings_job) {
if (j->error != 0)
if (j->error != 0) {
if (j == i->tar_job) {
if (j->error == ENOMEDIUM) /* HTTP 404 */
r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
else
r = log_error_errno(j->error, "Failed to retrieve image file.");
goto finish;
} else if (j == i->checksum_job) {
r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
goto finish;
} else if (j == i->signature_job)
log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
else if (j == i->settings_job)
log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
} else if (j->error != 0 && j != i->signature_job) {
if (j == i->checksum_job)
log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
else
log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
r = j->error;
goto finish;
assert("unexpected job");
}
/* This is invoked if either the download completed successfully, or the download was skipped because
@ -296,6 +347,8 @@ static void tar_pull_job_on_finished(PullJob *j) {
if (i->signature_job && i->signature_job->error != 0) {
VerificationStyle style;
assert(i->checksum_job);
r = verification_style_from_url(i->checksum_job->url, &style);
if (r < 0) {
log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
@ -306,19 +359,14 @@ static void tar_pull_job_on_finished(PullJob *j) {
* in per-directory verification mode, since only
* then the signature is detached, and thus a file
* of its own. */
log_error_errno(j->error, "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
r = i->signature_job->error;
r = log_error_errno(i->signature_job->error,
"Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
goto finish;
}
}
i->tar_job->disk_fd = safe_close(i->tar_job->disk_fd);
if (i->settings_job)
i->settings_job->disk_fd = safe_close(i->settings_job->disk_fd);
r = tar_pull_determine_path(i, NULL, &i->final_path);
if (r < 0)
goto finish;
pull_job_close_disk_fd(i->tar_job);
pull_job_close_disk_fd(i->settings_job);
if (i->tar_pid > 0) {
r = wait_for_terminate_and_check("tar", i->tar_pid, WAIT_LOG);
@ -337,6 +385,7 @@ static void tar_pull_job_on_finished(PullJob *j) {
tar_pull_report_progress(i, TAR_VERIFYING);
r = pull_verify(i->verify,
i->checksum,
i->tar_job,
i->checksum_job,
i->signature_job,
@ -346,60 +395,93 @@ static void tar_pull_job_on_finished(PullJob *j) {
/* verity_job = */ NULL);
if (r < 0)
goto finish;
}
if (i->flags & PULL_DIRECT) {
assert(!i->settings_job);
assert(i->local);
assert(!i->temp_path);
tar_pull_report_progress(i, TAR_FINALIZING);
r = import_mangle_os_tree(i->temp_path);
r = import_mangle_os_tree(i->local);
if (r < 0)
goto finish;
r = import_make_read_only(i->temp_path);
if (r < 0)
goto finish;
r = rename_noreplace(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path);
r = install_file(
AT_FDCWD, i->local,
AT_FDCWD, NULL,
(i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 |
(i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0) {
log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
log_error_errno(r, "Failed to finalize '%s': %m", i->local);
goto finish;
}
} else {
r = tar_pull_determine_path(i, NULL, &i->final_path);
if (r < 0)
goto finish;
i->temp_path = mfree(i->temp_path);
if (!i->tar_job->etag_exists) {
/* This is a new download, verify it, and move it into place */
if (i->settings_job &&
i->settings_job->error == 0) {
assert(i->temp_path);
assert(i->final_path);
/* Also move the settings file into place, if it exists. Note that we do so only if we also
* moved the tar file in place, to keep things strictly in sync. */
assert(i->settings_temp_path);
tar_pull_report_progress(i, TAR_FINALIZING);
/* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
* we should incorporate it in the file name if we can */
i->settings_path = mfree(i->settings_path);
r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
r = import_mangle_os_tree(i->temp_path);
if (r < 0)
goto finish;
r = import_make_read_only(i->settings_temp_path);
if (r < 0)
goto finish;
r = rename_noreplace(AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path);
r = install_file(
AT_FDCWD, i->temp_path,
AT_FDCWD, i->final_path,
INSTALL_READ_ONLY|
(i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
if (r < 0) {
log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
goto finish;
}
i->settings_temp_path = mfree(i->settings_temp_path);
i->temp_path = mfree(i->temp_path);
if (i->settings_job &&
i->settings_job->error == 0) {
/* Also move the settings file into place, if it exists. Note that we do so only if we also
* moved the tar file in place, to keep things strictly in sync. */
assert(i->settings_temp_path);
/* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
* we should incorporate it in the file name if we can */
i->settings_path = mfree(i->settings_path);
r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
if (r < 0)
goto finish;
r = install_file(
AT_FDCWD, i->settings_temp_path,
AT_FDCWD, i->settings_path,
INSTALL_READ_ONLY|
(i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
if (r < 0) {
log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
goto finish;
}
i->settings_temp_path = mfree(i->settings_temp_path);
}
}
tar_pull_report_progress(i, TAR_COPYING);
r = tar_pull_make_local_copy(i);
if (r < 0)
goto finish;
}
tar_pull_report_progress(i, TAR_COPYING);
r = tar_pull_make_local_copy(i);
if (r < 0)
goto finish;
r = 0;
finish:
@ -410,6 +492,7 @@ finish:
}
static int tar_pull_job_on_open_disk_tar(PullJob *j) {
const char *where;
TarPull *i;
int r;
@ -420,23 +503,39 @@ static int tar_pull_job_on_open_disk_tar(PullJob *j) {
assert(i->tar_job == j);
assert(i->tar_pid <= 0);
if (!i->temp_path) {
r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
if (r < 0)
return log_oom();
if (i->flags & PULL_DIRECT)
where = i->local;
else {
if (!i->temp_path) {
r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
if (r < 0)
return log_oom();
}
where = i->temp_path;
}
mkdir_parents_label(i->temp_path, 0700);
(void) mkdir_parents_label(where, 0700);
r = btrfs_subvol_make_fallback(i->temp_path, 0755);
if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE))
(void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
if (i->flags & PULL_BTRFS_SUBVOL)
r = btrfs_subvol_make_fallback(where, 0755);
else
r = mkdir(where, 0755) < 0 ? -errno : 0;
if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
* because in that case our temporary path collided */
r = 0;
if (r < 0)
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", i->temp_path);
if (r > 0) { /* actually btrfs subvol */
(void) import_assign_pool_quota_and_warn(i->image_root);
(void) import_assign_pool_quota_and_warn(i->temp_path);
return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */
if (!(i->flags & PULL_DIRECT))
(void) import_assign_pool_quota_and_warn(i->image_root);
(void) import_assign_pool_quota_and_warn(where);
}
j->disk_fd = import_fork_tar_x(i->temp_path, &i->tar_pid);
j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
if (j->disk_fd < 0)
return j->disk_fd;
@ -459,7 +558,7 @@ static int tar_pull_job_on_open_disk_settings(PullJob *j) {
return log_oom();
}
mkdir_parents_label(i->settings_temp_path, 0700);
(void) mkdir_parents_label(i->settings_temp_path, 0700);
j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
if (j->disk_fd < 0)
@ -484,19 +583,24 @@ int tar_pull_start(
const char *url,
const char *local,
PullFlags flags,
ImportVerify verify) {
ImportVerify verify,
const char *checksum) {
PullJob *j;
int r;
assert(i);
assert(verify < _IMPORT_VERIFY_MAX);
assert(verify >= 0);
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert(!(flags & ~PULL_FLAGS_MASK_TAR));
assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT));
assert(!(flags & PULL_SETTINGS) || !checksum);
if (!http_url_is_valid(url))
return -EINVAL;
if (local && !hostname_is_valid(local, 0))
if (local && !pull_validate_local(local, flags))
return -EINVAL;
if (i->tar_job)
@ -506,6 +610,10 @@ int tar_pull_start(
if (r < 0)
return r;
r = free_and_strdup(&i->checksum, checksum);
if (r < 0)
return r;
i->flags = flags;
i->verify = verify;
@ -516,52 +624,56 @@ int tar_pull_start(
i->tar_job->on_finished = tar_pull_job_on_finished;
i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
i->tar_job->on_progress = tar_pull_job_on_progress;
i->tar_job->calc_checksum = verify != IMPORT_VERIFY_NO;
i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
if (r < 0)
return r;
if (!FLAGS_SET(flags, PULL_DIRECT)) {
r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
if (r < 0)
return r;
}
/* Set up download of checksum/signature files */
r = pull_make_verification_jobs(&i->checksum_job, &i->signature_job, verify, url, i->glue, tar_pull_job_on_finished, i);
r = pull_make_verification_jobs(
&i->checksum_job,
&i->signature_job,
verify,
checksum,
url,
i->glue,
tar_pull_job_on_finished,
i);
if (r < 0)
return r;
/* Set up download job for the settings file (.nspawn) */
if (FLAGS_SET(flags, PULL_SETTINGS)) {
r = pull_make_auxiliary_job(&i->settings_job, url, tar_strip_suffixes, ".nspawn", i->glue, tar_pull_job_on_finished, i);
if (r < 0)
return r;
i->settings_job->on_open_disk = tar_pull_job_on_open_disk_settings;
i->settings_job->on_progress = tar_pull_job_on_progress;
i->settings_job->calc_checksum = verify != IMPORT_VERIFY_NO;
}
r = pull_job_begin(i->tar_job);
if (r < 0)
return r;
if (i->checksum_job) {
i->checksum_job->on_progress = tar_pull_job_on_progress;
i->checksum_job->on_not_found = pull_job_restart_with_sha256sum;
r = pull_job_begin(i->checksum_job);
r = pull_make_auxiliary_job(
&i->settings_job,
url,
tar_strip_suffixes,
".nspawn",
verify,
i->glue,
tar_pull_job_on_open_disk_settings,
tar_pull_job_on_finished,
i);
if (r < 0)
return r;
}
if (i->signature_job) {
i->signature_job->on_progress = tar_pull_job_on_progress;
FOREACH_POINTER(j,
i->tar_job,
i->checksum_job,
i->signature_job,
i->settings_job) {
r = pull_job_begin(i->signature_job);
if (r < 0)
return r;
}
if (!j)
continue;
if (i->settings_job) {
r = pull_job_begin(i->settings_job);
j->on_progress = tar_pull_job_on_progress;
j->sync = FLAGS_SET(flags, PULL_SYNC);
r = pull_job_begin(j);
if (r < 0)
return r;
}

2
src/import/pull-tar.h
View File

@ -16,4 +16,4 @@ TarPull* tar_pull_unref(TarPull *pull);
DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref);
int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify);
int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum);

406
src/import/pull.c
View File

@ -8,24 +8,87 @@
#include "alloc-util.h"
#include "discover-image.h"
#include "env-util.h"
#include "hexdecoct.h"
#include "hostname-util.h"
#include "import-common.h"
#include "import-util.h"
#include "io-util.h"
#include "main-func.h"
#include "parse-argument.h"
#include "parse-util.h"
#include "pull-raw.h"
#include "pull-tar.h"
#include "signal-util.h"
#include "string-util.h"
#include "terminal-util.h"
#include "verbs.h"
#include "web-util.h"
static const char *arg_image_root = "/var/lib/machines";
static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY;
static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC;
static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
static char *arg_checksum = NULL;
static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) {
log_notice("Transfer aborted.");
sd_event_exit(sd_event_source_get_event(s), EINTR);
STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep);
static int normalize_local(const char *local, const char *url, char **ret) {
_cleanup_free_ char *ll = NULL;
int r;
if (arg_pull_flags & PULL_DIRECT) {
if (!local)
log_debug("Writing downloaded data to STDOUT.");
else {
if (!path_is_absolute(local)) {
ll = path_join(arg_image_root, local);
if (!ll)
return log_oom();
local = ll;
}
if (!path_is_valid(local))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local path name '%s' is not valid.", local);
}
} else if (local) {
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
}
if (!ll && local) {
ll = strdup(local);
if (!ll)
return log_oom();
}
if (ll) {
if (arg_offset != UINT64_MAX)
log_info("Pulling '%s', saving at offset %" PRIu64 " in '%s'.", url, arg_offset, ll);
else
log_info("Pulling '%s', saving as '%s'.", url, ll);
} else
log_info("Pulling '%s'.", url);
*ret = TAKE_PTR(ll);
return 0;
}
@ -40,70 +103,57 @@ static void on_tar_finished(TarPull *pull, int error, void *userdata) {
}
static int pull_tar(int argc, char *argv[], void *userdata) {
_cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
_cleanup_(tar_pull_unrefp) TarPull *pull = NULL;
const char *url, *local;
_cleanup_free_ char *l = NULL, *ll = NULL;
int r;
url = argv[1];
if (!http_url_is_valid(url))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"URL '%s' is not valid.", url);
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
if (argc >= 3)
local = argv[2];
local = empty_or_dash_to_null(argv[2]);
else {
_cleanup_free_ char *l = NULL;
r = import_url_last_component(url, &l);
if (r < 0)
return log_error_errno(r, "Failed get final component of URL: %m");
return log_error_errno(r, "Failed to get final component of URL: %m");
local = l;
}
local = empty_or_dash_to_null(local);
if (local) {
r = tar_strip_suffixes(local, &ll);
r = tar_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
}
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported.");
if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else {
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
}
log_info("Pulling '%s', saving as '%s'.", url, local);
} else
log_info("Pulling '%s'.", url);
r = sd_event_default(&event);
r = normalize_local(local, url, &normalized);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
log_info("File system synchronization on completion is off.");
r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate puller: %m");
r = tar_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_TAR, arg_verify);
r = tar_pull_start(
pull,
url,
normalized,
arg_pull_flags & PULL_FLAGS_MASK_TAR,
arg_verify,
arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
@ -126,70 +176,55 @@ static void on_raw_finished(RawPull *pull, int error, void *userdata) {
}
static int pull_raw(int argc, char *argv[], void *userdata) {
_cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
_cleanup_free_ char *ll = NULL, *normalized = NULL;
_cleanup_(sd_event_unrefp) sd_event *event = NULL;
_cleanup_(raw_pull_unrefp) RawPull *pull = NULL;
const char *url, *local;
_cleanup_free_ char *l = NULL, *ll = NULL;
int r;
url = argv[1];
if (!http_url_is_valid(url))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"URL '%s' is not valid.", url);
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "URL '%s' is not valid.", url);
if (argc >= 3)
local = argv[2];
local = empty_or_dash_to_null(argv[2]);
else {
_cleanup_free_ char *l = NULL;
r = import_url_last_component(url, &l);
if (r < 0)
return log_error_errno(r, "Failed get final component of URL: %m");
return log_error_errno(r, "Failed to get final component of URL: %m");
local = l;
}
local = empty_or_dash_to_null(local);
if (local) {
r = raw_strip_suffixes(local, &ll);
r = raw_strip_suffixes(l, &ll);
if (r < 0)
return log_oom();
local = ll;
}
if (!hostname_is_valid(local, 0))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Local image name '%s' is not valid.",
local);
if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) {
r = image_find(IMAGE_MACHINE, local, NULL, NULL);
if (r < 0) {
if (r != -ENOENT)
return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local);
} else {
return log_error_errno(SYNTHETIC_ERRNO(EEXIST),
"Image '%s' already exists.",
local);
}
}
log_info("Pulling '%s', saving as '%s'.", url, local);
} else
log_info("Pulling '%s'.", url);
r = sd_event_default(&event);
r = normalize_local(local, url, &normalized);
if (r < 0)
return log_error_errno(r, "Failed to allocate event loop: %m");
return r;
assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0);
(void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL);
(void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL);
r = import_allocate_event_with_signals(&event);
if (r < 0)
return r;
r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
if (!FLAGS_SET(arg_pull_flags, PULL_SYNC))
log_info("File system synchronization on completion is off.");
r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event);
if (r < 0)
return log_error_errno(r, "Failed to allocate puller: %m");
r = raw_pull_start(pull, url, local, arg_pull_flags & PULL_FLAGS_MASK_RAW, arg_verify);
r = raw_pull_start(
pull,
url,
normalized,
arg_offset,
arg_size_max,
arg_pull_flags & PULL_FLAGS_MASK_RAW,
arg_verify,
arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
@ -203,23 +238,39 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
static int help(int argc, char *argv[], void *userdata) {
printf("%s [OPTIONS...] {COMMAND} ...\n\n"
"Download container or virtual machine images.\n\n"
printf("%1$s [OPTIONS...] {COMMAND} ...\n"
"\n%4$sDownload container or virtual machine images.%5$s\n"
"\n%2$sCommands:%3$s\n"
" tar URL [NAME] Download a TAR image\n"
" raw URL [NAME] Download a RAW image\n"
"\n%2$sOptions:%3$s\n"
" -h --help Show this help\n"
" --version Show package version\n"
" --force Force creation of image\n"
" --verify=MODE Verify downloaded image, one of: 'no',\n"
" 'checksum', 'signature'\n"
" 'checksum', 'signature' or literal SHA256 hash\n"
" --settings=BOOL Download settings file with image\n"
" --roothash=BOOL Download root hash file with image\n"
" --roothash-signature=BOOL\n"
" Download root hash signature file with image\n"
" --verity=BOOL Download verity file with image\n"
" --image-root=PATH Image root directory\n\n"
"Commands:\n"
" tar URL [NAME] Download a TAR image\n"
" raw URL [NAME] Download a RAW image\n",
program_invocation_short_name);
" --read-only Create a read-only image\n"
" --direct Download directly to specified file\n"
" --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n"
" instead of a directory\n"
" --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n"
" subvolume\n"
" --convert-qcow2=BOOL Controls whether to convert QCOW2 images to\n"
" regular disk images\n"
" --sync=BOOL Controls whether to sync() before completing\n"
" --offset=BYTES Offset to seek to in destination\n"
" --size-max=BYTES Maximum number of bytes to write to destination\n",
program_invocation_short_name,
ansi_underline(),
ansi_normal(),
ansi_highlight(),
ansi_normal());
return 0;
}
@ -235,6 +286,14 @@ static int parse_argv(int argc, char *argv[]) {
ARG_ROOTHASH,
ARG_ROOTHASH_SIGNATURE,
ARG_VERITY,
ARG_READ_ONLY,
ARG_DIRECT,
ARG_BTRFS_SUBVOL,
ARG_BTRFS_QUOTA,
ARG_CONVERT_QCOW2,
ARG_SYNC,
ARG_OFFSET,
ARG_SIZE_MAX,
};
static const struct option options[] = {
@ -247,6 +306,14 @@ static int parse_argv(int argc, char *argv[]) {
{ "roothash", required_argument, NULL, ARG_ROOTHASH },
{ "roothash-signature", required_argument, NULL, ARG_ROOTHASH_SIGNATURE },
{ "verity", required_argument, NULL, ARG_VERITY },
{ "read-only", no_argument, NULL, ARG_READ_ONLY },
{ "direct", no_argument, NULL, ARG_DIRECT },
{ "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL },
{ "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA },
{ "convert-qcow2", required_argument, NULL, ARG_CONVERT_QCOW2 },
{ "sync", required_argument, NULL, ARG_SYNC },
{ "offset", required_argument, NULL, ARG_OFFSET },
{ "size-max", required_argument, NULL, ARG_SIZE_MAX },
{}
};
@ -273,26 +340,51 @@ static int parse_argv(int argc, char *argv[]) {
arg_image_root = optarg;
break;
case ARG_VERIFY:
arg_verify = import_verify_from_string(optarg);
if (arg_verify < 0)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Invalid verification setting '%s'", optarg);
case ARG_VERIFY: {
ImportVerify v;
v = import_verify_from_string(optarg);
if (v < 0) {
_cleanup_free_ void *h = NULL;
char *hh;
size_t n;
/* If this is not a vaid verification mode, maybe it's a literally specified
* SHA256 hash? We can handle that too... */
r = unhexmem(optarg, (size_t) -1, &h, &n);
if (r < 0 || n == 0)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"Invalid verification setting: %s", optarg);
if (n != 32)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2);
hh = hexmem(h, n); /* bring into canonical (lowercase) form */
if (!hh)
return log_oom();
free_and_replace(arg_checksum, hh);
arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
arg_verify = _IMPORT_VERIFY_INVALID;
} else
arg_verify = v;
break;
}
case ARG_SETTINGS:
r = parse_boolean(optarg);
r = parse_boolean_argument("--settings=", optarg, NULL);
if (r < 0)
return log_error_errno(r, "Failed to parse --settings= parameter '%s': %m", optarg);
return r;
SET_FLAG(arg_pull_flags, PULL_SETTINGS, r);
break;
case ARG_ROOTHASH:
r = parse_boolean(optarg);
r = parse_boolean_argument("--roothash=", optarg, NULL);
if (r < 0)
return log_error_errno(r, "Failed to parse --roothash= parameter '%s': %m", optarg);
return r;
SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r);
@ -302,21 +394,88 @@ static int parse_argv(int argc, char *argv[]) {
break;
case ARG_ROOTHASH_SIGNATURE:
r = parse_boolean(optarg);
r = parse_boolean_argument("--roothash-signature=", optarg, NULL);
if (r < 0)
return log_error_errno(r, "Failed to parse --roothash-signature= parameter '%s': %m", optarg);
return r;
SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r);
break;
case ARG_VERITY:
r = parse_boolean(optarg);
r = parse_boolean_argument("--verity=", optarg, NULL);
if (r < 0)
return log_error_errno(r, "Failed to parse --verity= parameter '%s': %m", optarg);
return r;
SET_FLAG(arg_pull_flags, PULL_VERITY, r);
break;
case ARG_READ_ONLY:
arg_pull_flags |= PULL_READ_ONLY;
break;
case ARG_DIRECT:
arg_pull_flags |= PULL_DIRECT;
arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY);
break;
case ARG_BTRFS_SUBVOL:
r = parse_boolean_argument("--btrfs-subvol=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
break;
case ARG_BTRFS_QUOTA:
r = parse_boolean_argument("--btrfs-quota=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
break;
case ARG_CONVERT_QCOW2:
r = parse_boolean_argument("--convert-qcow2=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r);
break;
case ARG_SYNC:
r = parse_boolean_argument("--sync=", optarg, NULL);
if (r < 0)
return r;
SET_FLAG(arg_pull_flags, PULL_SYNC, r);
break;
case ARG_OFFSET: {
uint64_t u;
r = safe_atou64(optarg, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --offset= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --offset= switch too large: %s", optarg);
arg_offset = u;
break;
}
case ARG_SIZE_MAX: {
uint64_t u;
r = parse_size(optarg, 1024, &u);
if (r < 0)
return log_error_errno(r, "Failed to parse --size-max= argument: %s", optarg);
if (!FILE_SIZE_VALID(u))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Argument to --size-max= switch too large: %s", optarg);
arg_size_max = u;
break;
}
case '?':
return -EINVAL;
@ -324,9 +483,46 @@ static int parse_argv(int argc, char *argv[]) {
assert_not_reached();
}
/* Make sure offset+size is still in the valid range if both set */
if (arg_offset != UINT64_MAX && arg_size_max != UINT64_MAX &&
((arg_size_max > (UINT64_MAX - arg_offset)) ||
!FILE_SIZE_VALID(arg_offset + arg_size_max)))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range.");
if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded.");
return 1;
}
static void parse_env(void) {
int r;
/* Let's make these relatively low-level settings also controllable via env vars. User can then set
* them for systemd-importd.service if they like to tweak behaviour */
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL");
if (r >= 0)
SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m");
r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA");
if (r >= 0)
SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m");
r = getenv_bool("SYSTEMD_IMPORT_SYNC");
if (r >= 0)
SET_FLAG(arg_pull_flags, PULL_SYNC, r);
else if (r != -ENXIO)
log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m");
}
static int pull_main(int argc, char *argv[]) {
static const Verb verbs[] = {
{ "help", VERB_ANY, VERB_ANY, 0, help },
@ -345,6 +541,8 @@ static int run(int argc, char *argv[]) {
log_parse_environment();
log_open();
parse_env();
r = parse_argv(argc, argv);
if (r <= 0)
return r;

269
src/shared/install-file.c Normal file
View File

@ -0,0 +1,269 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <sys/ioctl.h>
#include "btrfs-util.h"
#include "chattr-util.h"
#include "errno-util.h"
#include "fd-util.h"
#include "fs-util.h"
#include "install-file.h"
#include "missing_syscall.h"
#include "rm-rf.h"
int fs_make_very_read_only(int fd) {
struct stat st;
int r;
assert(fd >= 0);
/* Tries to make the specified fd "comprehensively" read-only. Primary usecase for this is OS images,
* i.e. either loopback files or larger directory hierarchies. Depending on the inode type and
* backing file system this means something different:
*
* 1. If the fd refers to a btrfs subvolume we'll mark it read-only as a whole
* 2. If the fd refers to any other directory we'll set the FS_IMMUTABLE_FL flag on it
* 3. If the fd refers to a regular file we'll drop the w bits.
* 4. If the fd refers to a block device, use BLKROSET to set read-only state
*
* You might wonder why not drop the x bits for directories. That's because we want to guarantee that
* everything "inside" the image remains largely the way it is, in case you mount it. And since the
* mode of the root dir of the image is pretty visible we don't want to modify it. btrfs subvol flags
* and the FS_IMMUTABLE_FL otoh are much less visible. Changing the mode of regular files should be
* OK though, since after all this is supposed to be used for disk images, i.e. the fs in the disk
* image doesn't make the mode of the loopback file it is stored in visible. */
if (fstat(fd, &st) < 0)
return -errno;
switch (st.st_mode & S_IFMT) {
case S_IFDIR:
if (btrfs_might_be_subvol(&st)) {
r = btrfs_subvol_set_read_only_fd(fd, true);
if (r >= 0)
return 0;
if (!ERRNO_IS_NOT_SUPPORTED(r) && r != -EINVAL)
return r;
}
r = chattr_fd(fd, FS_IMMUTABLE_FL, FS_IMMUTABLE_FL, NULL);
if (r < 0)
return r;
break;
case S_IFREG:
if ((st.st_mode & 0222) != 0)
if (fchmod(fd, st.st_mode & 07555) < 0)
return -errno;
break;
case S_IFBLK: {
int ro = 1;
if (ioctl(fd, BLKROSET, &ro) < 0)
return -errno;
break;
}
default:
return -EBADFD;
}
return 0;
}
static int unlinkat_maybe_dir(int dirfd, const char *pathname) {
/* Invokes unlinkat() for regular files first, and if this fails with EISDIR tries again with
* AT_REMOVEDIR */
if (unlinkat(dirfd, pathname, 0) < 0) {
if (errno != EISDIR)
return -errno;
if (unlinkat(dirfd, pathname, AT_REMOVEDIR) < 0)
return -errno;
}
return 0;
}
int install_file(int source_atfd, const char *source_name,
int target_atfd, const char *target_name,
InstallFileFlags flags) {
_cleanup_close_ int rofd = -1;
int r;
/* Moves a file or directory tree into place, with some bells and whistles:
*
* 1. Optionally syncs before/after to ensure file installation can be used as barrier
* 2. Optionally marks the file/directory read-only using fs_make_very_read_only()
* 3. Optionally operates in replacing or in non-replacing mode.
* 4. If it replaces will remove the old tree if needed.
*/
assert(source_atfd >= 0 || source_atfd == AT_FDCWD);
assert(source_name);
assert(target_atfd >= 0 || target_atfd == AT_FDCWD);
/* If target_name is specified as NULL no renaming takes place. Instead it is assumed the file is
* already in place, and only the syncing/read-only marking shall be applied. Note that with
* target_name=NULL and flags=0 this call is a NOP */
if ((flags & (INSTALL_FSYNC|INSTALL_FSYNC_FULL|INSTALL_SYNCFS|INSTALL_READ_ONLY)) != 0) {
_cleanup_close_ int pfd = -1;
struct stat st;
/* Open an O_PATH fd for the source if we need to sync things or mark things read only. */
pfd = openat(source_atfd, source_name, O_PATH|O_CLOEXEC|O_NOFOLLOW);
if (pfd < 0)
return -errno;
if (fstat(pfd, &st) < 0)
return -errno;
switch (st.st_mode & S_IFMT) {
case S_IFREG: {
_cleanup_close_ int regfd = -1;
regfd = fd_reopen(pfd, O_RDONLY|O_CLOEXEC);
if (regfd < 0)
return regfd;
if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
/* If this is just a regular file (as oppose to a fully populated directory)
* let's downgrade INSTALL_SYNCFS to INSTALL_FSYNC_FULL, after all this is
* going to be a single inode we install */
r = fsync_full(regfd);
if (r < 0)
return r;
} else if (flags & INSTALL_FSYNC) {
if (fsync(regfd) < 0)
return -errno;
}
if (flags & INSTALL_READ_ONLY)
rofd = TAKE_FD(regfd);
break;
}
case S_IFDIR: {
_cleanup_close_ int dfd = -1;
dfd = fd_reopen(pfd, O_RDONLY|O_DIRECTORY|O_CLOEXEC);
if (dfd < 0)
return dfd;
if (flags & INSTALL_SYNCFS) {
if (syncfs(dfd) < 0)
return -errno;
} else if (flags & INSTALL_FSYNC_FULL) {
r = fsync_full(dfd);
if (r < 0)
return r;
} else if (flags & INSTALL_FSYNC) {
if (fsync(dfd) < 0)
return -errno;
}
if (flags & INSTALL_READ_ONLY)
rofd = TAKE_FD(dfd);
break;
}
default:
/* Other inodes: char/block device inodes, fifos, symlinks, sockets don't need
* syncing themselves, as they only exist in the directory, and have no contents on
* disk */
if (target_name && (flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
r = fsync_directory_of_file(pfd);
if (r < 0)
return r;
}
break;
}
}
if (target_name) {
/* Rename the file */
if (flags & INSTALL_REPLACE) {
/* First, try a simple renamat(), maybe that's enough */
if (renameat(source_atfd, source_name, target_atfd, target_name) < 0) {
_cleanup_close_ int dfd = -1;
if (!IN_SET(errno, EEXIST, ENOTDIR, ENOTEMPTY, EISDIR, EBUSY))
return -errno;
/* Hmm, the target apparently existed already. Let's try to use
* RENAME_EXCHANGE. But let's first open the inode if it's a directory, so
* that we can later remove its contents if it's a directory. Why do this
* before the rename()? Mostly because if we have trouble opening the thing
* we want to know before we start actually modifying the file system. */
dfd = openat(target_atfd, target_name, O_RDONLY|O_DIRECTORY|O_CLOEXEC, 0);
if (dfd < 0 && errno != ENOTDIR)
return -errno;
if (renameat2(source_atfd, source_name, target_atfd, target_name, RENAME_EXCHANGE) < 0) {
if (!ERRNO_IS_NOT_SUPPORTED(errno) && errno != EINVAL)
return -errno;
/* The exchange didn't work, let's remove the target first, and try again */
if (dfd >= 0)
(void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL);
r = unlinkat_maybe_dir(target_atfd, target_name);
if (r < 0)
return log_debug_errno(r, "Failed to remove target directory: %m");
if (renameat(source_atfd, source_name, target_atfd, target_name) < 0)
return -errno;
} else {
/* The exchange worked, hence let's remove the source (i.e. the old target) */
if (dfd >= 0)
(void) rm_rf_children(TAKE_FD(dfd), REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD, NULL);
r = unlinkat_maybe_dir(source_atfd, source_name);
if (r < 0)
return log_debug_errno(r, "Failed to remove replaced target directory: %m");
}
}
} else {
r = rename_noreplace(source_atfd, source_name, target_atfd, target_name);
if (r < 0)
return r;
}
}
if (rofd >= 0) {
r = fs_make_very_read_only(rofd);
if (r < 0)
return r;
}
if ((flags & (INSTALL_FSYNC_FULL|INSTALL_SYNCFS)) != 0) {
if (target_name)
r = fsync_parent_at(target_atfd, target_name);
else
r = fsync_parent_at(source_atfd, source_name);
if (r < 0)
return r;
}
return 0;
}

14
src/shared/install-file.h Normal file
View File

@ -0,0 +1,14 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#pragma once
int fs_make_very_read_only(int fd);
typedef enum InstallFileFlags {
INSTALL_REPLACE = 1 << 0, /* Replace an existing inode */
INSTALL_READ_ONLY = 1 << 1, /* Call fs_make_very_read_only() to make the inode comprehensively read-only */
INSTALL_FSYNC = 1 << 2, /* fsync() file contents before moving file in */
INSTALL_FSYNC_FULL = 1 << 3, /* like INSTALL_FSYNC, but also fsync() parent dir before+after moving file in */
INSTALL_SYNCFS = 1 << 4, /* syncfs() before moving file in, fsync() parent dir after moving file in */
} InstallFileFlags;
int install_file(int source_atfd, const char *source_name, int target_atfd, const char *target_name, InstallFileFlags flags);

2
src/shared/meson.build
View File

@ -152,6 +152,8 @@ shared_sources = files('''
import-util.c
import-util.h
initreq.h
install-file.c
install-file.h
install-printf.c
install-printf.h
install.c

2
src/test/meson.build
View File

@ -215,6 +215,8 @@ tests += [
[['src/test/test-fs-util.c']],
[['src/test/test-install-file.c']],
[['src/test/test-umask-util.c']],
[['src/test/test-proc-cmdline.c']],

72
src/test/test-install-file.c Normal file
View File

@ -0,0 +1,72 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include "fileio.h"
#include "install-file.h"
#include "path-util.h"
#include "rm-rf.h"
#include "tests.h"
#include "tmpfile-util.h"
#include "umask-util.h"
static void test_install_file(void) {
_cleanup_(rm_rf_physical_and_freep) char *p = NULL;
_cleanup_free_ char *a = NULL, *b = NULL, *c = NULL;
struct stat stat1, stat2;
log_info("/* %s */", __func__);
assert_se(mkdtemp_malloc(NULL, &p) >= 0);
assert_se(a = path_join(p, "foo"));
assert_se(b = path_join(p, "bar"));
RUN_WITH_UMASK(0077)
assert_se(write_string_file(a, "wups", WRITE_STRING_FILE_CREATE) >= 0);
assert_se(lstat(a, &stat1) >= 0);
assert_se(S_ISREG(stat1.st_mode));
assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, 0) >= 0);
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC) >= 0);
assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0);
assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL) == -EEXIST);
assert_se(install_file(AT_FDCWD, a, AT_FDCWD, b, INSTALL_FSYNC_FULL|INSTALL_REPLACE) >= 0);
assert_se(stat(b, &stat2) >= 0);
assert_se(stat1.st_dev == stat2.st_dev);
assert_se(stat1.st_ino == stat2.st_ino);
assert_se((stat2.st_mode & 0222) != 0); /* writable */
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE|INSTALL_READ_ONLY) >= 0);
assert_se(stat(a, &stat2) >= 0);
assert_se(stat1.st_dev == stat2.st_dev);
assert_se(stat1.st_ino == stat2.st_ino);
assert_se((stat2.st_mode & 0222) == 0); /* read-only */
assert_se(mkdir(b, 0755) >= 0);
assert_se(c = path_join(b, "dir"));
assert_se(mkdir(c, 0755) >= 0);
free(c);
assert_se(c = path_join(b, "reg"));
assert_se(mknod(c, S_IFREG|0755, 0) >= 0);
free(c);
assert_se(c = path_join(b, "fifo"));
assert_se(mknod(c, S_IFIFO|0755, 0) >= 0);
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST);
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0);
assert_se(write_string_file(b, "ttss", WRITE_STRING_FILE_CREATE) >= 0);
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL) == -EEXIST);
assert_se(install_file(AT_FDCWD, b, AT_FDCWD, a, INSTALL_FSYNC_FULL|INSTALL_REPLACE) == 0);
}
int main(int argc, char *argv[]) {
test_setup_logging(LOG_INFO);
test_install_file();
return 0;
}