From 355f8438ef4fe2720d25e8241aa76797b7029522 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 Jan 2016 14:41:27 -0500 Subject: [PATCH] Add an `export` builtin, and API to write to libarchive At the moment I'm looking at using rpm-ostree to manage RPM inputs which can then be converted into Docker images. It's most convenient if we can stream directly out of libostree rather than doing a checkout + tar combination. There are also backup/debugging etc. reasons to implement `export` as well. --- Makefile-man.am | 2 +- Makefile-ostree.am | 1 + Makefile-tests.am | 1 + src/libostree/ostree-repo-libarchive.c | 249 +++++++++++++++++++++++++ src/libostree/ostree-repo.h | 24 +++ src/ostree/main.c | 1 + src/ostree/ot-builtin-diff.c | 9 +- src/ostree/ot-builtin-export.c | 148 +++++++++++++++ src/ostree/ot-builtins.h | 1 + tests/test-export.sh | 50 +++++ 10 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 src/ostree/ot-builtin-export.c create mode 100755 tests/test-export.sh diff --git a/Makefile-man.am b/Makefile-man.am index a6090bf4..615bf0f0 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -19,7 +19,7 @@ if ENABLE_MAN -man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 +man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 if BUILDOPT_FUSE man1_files += rofiles-fuse.1 diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 76fb26f3..ab4485c6 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-checksum.c \ src/ostree/ot-builtin-commit.c \ src/ostree/ot-builtin-diff.c \ + src/ostree/ot-builtin-export.c \ src/ostree/ot-builtin-fsck.c \ src/ostree/ot-builtin-gpg-sign.c \ src/ostree/ot-builtin-init.c \ diff --git a/Makefile-tests.am b/Makefile-tests.am index b0466840..292699ef 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -27,6 +27,7 @@ testfiles = test-basic \ test-remote-add \ test-remote-gpg-import \ test-commit-sign \ + test-export \ test-help \ test-libarchive \ test-pull-archive-z \ diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index e86e4026..3b1b0b83 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -24,6 +24,7 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-repo-file.h" #include "ostree-mutable-tree.h" #ifdef HAVE_LIBARCHIVE @@ -356,3 +357,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, return FALSE; #endif } + +#ifdef HAVE_LIBARCHIVE + +static gboolean +file_to_archive_entry_common (GFile *root, + OstreeRepoArchiveOptions *opts, + GFile *path, + GFileInfo *file_info, + struct archive_entry *entry, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *pathstr = g_file_get_relative_path (root, path); + g_autoptr(GVariant) xattrs = NULL; + time_t ts = (time_t) opts->timestamp_secs; + + if (pathstr && !pathstr[0]) + { + g_free (pathstr); + pathstr = g_strdup ("."); + } + + archive_entry_update_pathname_utf8 (entry, pathstr); + archive_entry_set_ctime (entry, ts, 0); + archive_entry_set_mtime (entry, ts, 0); + archive_entry_set_atime (entry, ts, 0); + archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid")); + archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid")); + archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); + + if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error)) + goto out; + + if (!opts->disable_xattrs) + { + int i, n; + + n = g_variant_n_children (xattrs); + for (i = 0; i < n; i++) + { + const guint8* name; + g_autoptr(GVariant) value = NULL; + const guint8* value_data; + gsize value_len; + + g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value); + value_data = g_variant_get_fixed_array (value, &value_len, 1); + + archive_entry_xattr_add_entry (entry, (char*)name, + (char*) value_data, value_len); + } + } + + ret = TRUE; + out: + return ret; +} + +static gboolean +write_header_free_entry (struct archive *a, + struct archive_entry **entryp, + GError **error) +{ + struct archive_entry *entry = *entryp; + gboolean ret = FALSE; + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + ret = TRUE; + out: + archive_entry_free (entry); + *entryp = NULL; + return ret; +} + +static gboolean +write_directory_to_libarchive_recurse (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + GFile *root, + GFile *dir, + struct archive *a, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autoptr(GFileInfo) dir_info = NULL; + g_autoptr(GFileEnumerator) dir_enum = NULL; + struct archive_entry *entry = NULL; + + dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_info) + goto out; + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error)) + goto out; + if (!write_header_free_entry (a, &entry, error)) + goto out; + + dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_enum) + goto out; + + while (TRUE) + { + GFileInfo *file_info; + GFile *path; + + if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path, + cancellable, error)) + goto out; + if (file_info == NULL) + break; + + /* First, handle directories recursively */ + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + { + if (!write_directory_to_libarchive_recurse (self, opts, root, path, a, + cancellable, error)) + goto out; + + /* Go to the next entry */ + continue; + } + + /* Past here, should be a regular file or a symlink */ + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error)) + goto out; + + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_SYMBOLIC_LINK: + { + archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info)); + if (!write_header_free_entry (a, &entry, error)) + goto out; + } + break; + case G_FILE_TYPE_REGULAR: + { + guint8 buf[8192]; + g_autoptr(GInputStream) file_in = NULL; + g_autoptr(GFileInfo) file_info = NULL; + const char *checksum; + + checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path); + + if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL, + cancellable, error)) + goto out; + + archive_entry_set_size (entry, g_file_info_get_size (file_info)); + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + while (TRUE) + { + gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf), + cancellable, error); + if (bytes_read < 0) + goto out; + if (bytes_read == 0) + break; + + { ssize_t r = archive_write_data (a, buf, bytes_read); + if (r != bytes_read) + { + propagate_libarchive_error (error, a); + g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", bytes_read, r); + goto out; + } + } + } + + if (archive_write_finish_entry (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + archive_entry_free (entry); + entry = NULL; + } + break; + default: + g_assert_not_reached (); + } + } + + ret = TRUE; + out: + if (entry) + archive_entry_free (entry); + return ret; +} +#endif + +/** + * ostree_repo_write_tree_to_archive: + * @self: An #OstreeRepo + * @opts: Options controlling conversion + * @root: An #OstreeRepoFile for the base directory + * @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_write_tree_to_archive (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, + GCancellable *cancellable, + GError **error) +{ +#ifdef HAVE_LIBARCHIVE + gboolean ret = FALSE; + struct archive *a = archive; + + if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root, + a, cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + return FALSE; +#endif +} diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d4d0f418..64e8a028 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -441,6 +441,23 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeRepoWriteArchiveOptions: + * + * An extensible options structure controlling archive creation. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_write_tree_to_archive(). + */ +typedef struct { + guint disable_xattrs : 1; + guint reserved : 31; + + guint64 timestamp_secs; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoArchiveOptions; + gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self, GFile *archive, OstreeMutableTree *mtree, @@ -449,6 +466,13 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); +gboolean ostree_repo_write_tree_to_archive (OstreeRepo *self, + OstreeRepoArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, /* Really struct archive * */ + GCancellable *cancellable, + GError **error); + gboolean ostree_repo_write_mtree (OstreeRepo *self, OstreeMutableTree *mtree, GFile **out_file, diff --git a/src/ostree/main.c b/src/ostree/main.c index 99d7d916..eff3082d 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -40,6 +40,7 @@ static OstreeCommand commands[] = { { "commit", ostree_builtin_commit }, { "config", ostree_builtin_config }, { "diff", ostree_builtin_diff }, + { "export", ostree_builtin_export }, { "fsck", ostree_builtin_fsck }, { "gpg-sign", ostree_builtin_gpg_sign }, { "init", ostree_builtin_init }, diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c index b5e0c5a8..a23ed83f 100644 --- a/src/ostree/ot-builtin-diff.c +++ b/src/ostree/ot-builtin-diff.c @@ -29,10 +29,12 @@ static gboolean opt_stats; static gboolean opt_fs_diff; +static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "stats", 0, 0, G_OPTION_ARG_NONE, &opt_stats, "Print various statistics", NULL }, { "fs-diff", 0, 0, G_OPTION_ARG_NONE, &opt_fs_diff, "Print filesystem diff", NULL }, + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, { NULL } }; @@ -162,6 +164,11 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_fs_diff) { + OstreeDiffFlags diff_flags = OSTREE_DIFF_FLAGS_NONE; + + if (opt_no_xattrs) + diff_flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + if (!parse_file_or_commit (repo, src, &srcf, cancellable, error)) goto out; if (!parse_file_or_commit (repo, target, &targetf, cancellable, error)) @@ -171,7 +178,7 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** removed = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); added = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, srcf, targetf, modified, removed, added, cancellable, error)) + if (!ostree_diff_dirs (diff_flags, srcf, targetf, modified, removed, added, cancellable, error)) goto out; ostree_diff_print (srcf, targetf, modified, removed, added); diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c new file mode 100644 index 00000000..2d350b8c --- /dev/null +++ b/src/ostree/ot-builtin-export.c @@ -0,0 +1,148 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-main.h" +#include "ot-builtins.h" +#include "ostree.h" +#include "ostree-repo-file.h" +#include "otutil.h" + +#ifdef HAVE_LIBARCHIVE +#include +#include +#endif + +static char *opt_output_path; +static gboolean opt_no_xattrs; + +static GOptionEntry options[] = { + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, + { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, + { NULL } +}; + +#ifdef HAVE_LIBARCHIVE + +static void +propagate_libarchive_error (GError **error, + struct archive *a) +{ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "%s", archive_error_string (a)); +} + +#endif + +gboolean +ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + gboolean ret = FALSE; + const char *rev; + g_autoptr(GFile) root = NULL; + g_autofree char *commit = NULL; + g_autoptr(GVariant) commit_data = NULL; + struct archive *a; + OstreeRepoArchiveOptions opts = { 0, }; + + context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format"); + + if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + +#ifdef HAVE_LIBARCHIVE + + if (argc <= 1) + { + ot_util_usage_error (context, "A COMMIT argument is required", error); + goto out; + } + rev = argv[1]; + + a = archive_write_new (); + /* Yes, this is hardcoded for now. There is + * archive_write_set_format_filter_by_ext() but it's fairly magic. + * Many programs have support now for GNU tar, so should be a good + * default. I also don't want to lock us into everything libarchive + * supports. + */ + if (archive_write_set_format_gnutar (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (archive_write_add_filter_none (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (opt_output_path) + { + if (archive_write_open_filename (a, opt_output_path) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + else + { + if (archive_write_open_FILE (a, stdout) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + + if (opt_no_xattrs) + opts.disable_xattrs = TRUE; + + if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error)) + goto out; + + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_data, error)) + goto out; + + opts.timestamp_secs = ostree_commit_get_timestamp (commit_data); + + if (!ostree_repo_write_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, + cancellable, error)) + goto out; + + if (archive_write_close (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + goto out; +#endif + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index 95262ec4..1c862925 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -35,6 +35,7 @@ BUILTINPROTO(checkout); BUILTINPROTO(checksum); BUILTINPROTO(commit); BUILTINPROTO(diff); +BUILTINPROTO(export); BUILTINPROTO(gpg_sign); BUILTINPROTO(init); BUILTINPROTO(log); diff --git a/tests/test-export.sh b/tests/test-export.sh new file mode 100755 index 00000000..18f2f7ae --- /dev/null +++ b/tests/test-export.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_test_repository "archive-z2" + +echo '1..2' + +$OSTREE checkout test2 test2-co +$OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co +rm test2-co -rf + +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs -o test2.tar +mkdir t +(cd t && tar xf ../test2.tar) +ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export gnutar diff (no xattrs)' + +cd ${test_tmpdir} +${OSTREE} 'export' test2 -o test2.tar +${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar +ostree --repo=repo diff test2 test2-from-tar +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export import' +