shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-01-03 05:18:24 +03:00
Code Issues Projects Releases Wiki Activity

Add an export builtin, and API to write to libarchive

Browse Source 
At the moment I'm looking at using rpm-ostree to manage RPM inputs
which can then be converted into Docker images.  It's most convenient
if we can stream directly out of libostree rather than doing a
checkout + tar combination.

There are also backup/debugging etc. reasons to implement `export` as
well.
This commit is contained in:
Colin Walters 2016-01-28 14:41:27 -05:00
parent e9ccdd2d00
commit 355f8438ef
10 changed files with 484 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile-man.am
View File

@ -19,7 +19,7 @@
if ENABLE_MAN
man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
if BUILDOPT_FUSE
man1_files += rofiles-fuse.1

1
Makefile-ostree.am
View File

@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-checksum.c \
src/ostree/ot-builtin-commit.c \
src/ostree/ot-builtin-diff.c \
src/ostree/ot-builtin-export.c \
src/ostree/ot-builtin-fsck.c \
src/ostree/ot-builtin-gpg-sign.c \
src/ostree/ot-builtin-init.c \

1
Makefile-tests.am
View File

@ -27,6 +27,7 @@ testfiles = test-basic \
test-remote-add \
test-remote-gpg-import \
test-commit-sign \
test-export \
test-help \
test-libarchive \
test-pull-archive-z \

249
src/libostree/ostree-repo-libarchive.c
View File

@ -24,6 +24,7 @@
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#include "ostree-repo-file.h"
#include "ostree-mutable-tree.h"
#ifdef HAVE_LIBARCHIVE
@ -356,3 +357,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self,
return FALSE;
#endif
}
#ifdef HAVE_LIBARCHIVE
static gboolean
file_to_archive_entry_common (GFile *root,
OstreeRepoArchiveOptions *opts,
GFile *path,
GFileInfo *file_info,
struct archive_entry *entry,
GError **error)
{
gboolean ret = FALSE;
g_autofree char *pathstr = g_file_get_relative_path (root, path);
g_autoptr(GVariant) xattrs = NULL;
time_t ts = (time_t) opts->timestamp_secs;
if (pathstr && !pathstr[0])
{
g_free (pathstr);
pathstr = g_strdup (".");
}
archive_entry_update_pathname_utf8 (entry, pathstr);
archive_entry_set_ctime (entry, ts, 0);
archive_entry_set_mtime (entry, ts, 0);
archive_entry_set_atime (entry, ts, 0);
archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid"));
archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error))
goto out;
if (!opts->disable_xattrs)
{
int i, n;
n = g_variant_n_children (xattrs);
for (i = 0; i < n; i++)
{
const guint8* name;
g_autoptr(GVariant) value = NULL;
const guint8* value_data;
gsize value_len;
g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value);
value_data = g_variant_get_fixed_array (value, &value_len, 1);
archive_entry_xattr_add_entry (entry, (char*)name,
(char*) value_data, value_len);
}
}
ret = TRUE;
out:
return ret;
}
static gboolean
write_header_free_entry (struct archive *a,
struct archive_entry **entryp,
GError **error)
{
struct archive_entry *entry = *entryp;
gboolean ret = FALSE;
if (archive_write_header (a, entry) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
ret = TRUE;
out:
archive_entry_free (entry);
*entryp = NULL;
return ret;
}
static gboolean
write_directory_to_libarchive_recurse (OstreeRepo *self,
OstreeRepoArchiveOptions *opts,
GFile *root,
GFile *dir,
struct archive *a,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFileInfo) dir_info = NULL;
g_autoptr(GFileEnumerator) dir_enum = NULL;
struct archive_entry *entry = NULL;
dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_info)
goto out;
entry = archive_entry_new2 (a);
if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error))
goto out;
if (!write_header_free_entry (a, &entry, error))
goto out;
dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_enum)
goto out;
while (TRUE)
{
GFileInfo *file_info;
GFile *path;
if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path,
cancellable, error))
goto out;
if (file_info == NULL)
break;
/* First, handle directories recursively */
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
{
if (!write_directory_to_libarchive_recurse (self, opts, root, path, a,
cancellable, error))
goto out;
/* Go to the next entry */
continue;
}
/* Past here, should be a regular file or a symlink */
entry = archive_entry_new2 (a);
if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error))
goto out;
switch (g_file_info_get_file_type (file_info))
{
case G_FILE_TYPE_SYMBOLIC_LINK:
{
archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info));
if (!write_header_free_entry (a, &entry, error))
goto out;
}
break;
case G_FILE_TYPE_REGULAR:
{
guint8 buf[8192];
g_autoptr(GInputStream) file_in = NULL;
g_autoptr(GFileInfo) file_info = NULL;
const char *checksum;
checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path);
if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL,
cancellable, error))
goto out;
archive_entry_set_size (entry, g_file_info_get_size (file_info));
if (archive_write_header (a, entry) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
while (TRUE)
{
gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf),
cancellable, error);
if (bytes_read < 0)
goto out;
if (bytes_read == 0)
break;
{ ssize_t r = archive_write_data (a, buf, bytes_read);
if (r != bytes_read)
{
propagate_libarchive_error (error, a);
g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", bytes_read, r);
goto out;
}
}
}
if (archive_write_finish_entry (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
archive_entry_free (entry);
entry = NULL;
}
break;
default:
g_assert_not_reached ();
}
}
ret = TRUE;
out:
if (entry)
archive_entry_free (entry);
return ret;
}
#endif
/**
* ostree_repo_write_tree_to_archive:
* @self: An #OstreeRepo
* @opts: Options controlling conversion
* @root: An #OstreeRepoFile for the base directory
* @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers
* @cancellable: Cancellable
* @error: Error
*
* Import an archive file @archive into the repository, and write its
* file structure to @mtree.
*/
gboolean
ostree_repo_write_tree_to_archive (OstreeRepo *self,
OstreeRepoArchiveOptions *opts,
OstreeRepoFile *root,
void *archive,
GCancellable *cancellable,
GError **error)
{
#ifdef HAVE_LIBARCHIVE
gboolean ret = FALSE;
struct archive *a = archive;
if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root,
a, cancellable, error))
goto out;
ret = TRUE;
out:
return ret;
#else
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree is not compiled with libarchive support");
return FALSE;
#endif
}

24
src/libostree/ostree-repo.h
View File

@ -441,6 +441,23 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
/**
* OstreeRepoWriteArchiveOptions:
*
* An extensible options structure controlling archive creation. Ensure that
* you have entirely zeroed the structure, then set just the desired
* options. This is used by ostree_repo_write_tree_to_archive().
*/
typedef struct {
guint disable_xattrs : 1;
guint reserved : 31;
guint64 timestamp_secs;
guint unused_uint[8];
gpointer unused_ptrs[8];
} OstreeRepoArchiveOptions;
gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self,
GFile *archive,
OstreeMutableTree *mtree,
@ -449,6 +466,13 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_write_tree_to_archive (OstreeRepo *self,
OstreeRepoArchiveOptions *opts,
OstreeRepoFile *root,
void *archive, /* Really struct archive * */
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_write_mtree (OstreeRepo *self,
OstreeMutableTree *mtree,
GFile **out_file,

1
src/ostree/main.c
View File

@ -40,6 +40,7 @@ static OstreeCommand commands[] = {
{ "commit", ostree_builtin_commit },
{ "config", ostree_builtin_config },
{ "diff", ostree_builtin_diff },
{ "export", ostree_builtin_export },
{ "fsck", ostree_builtin_fsck },
{ "gpg-sign", ostree_builtin_gpg_sign },
{ "init", ostree_builtin_init },

9
src/ostree/ot-builtin-diff.c
View File

@ -29,10 +29,12 @@
static gboolean opt_stats;
static gboolean opt_fs_diff;
static gboolean opt_no_xattrs;
static GOptionEntry options[] = {
{ "stats", 0, 0, G_OPTION_ARG_NONE, &opt_stats, "Print various statistics", NULL },
{ "fs-diff", 0, 0, G_OPTION_ARG_NONE, &opt_fs_diff, "Print filesystem diff", NULL },
{ "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL },
{ NULL }
};
@ -162,6 +164,11 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **
if (opt_fs_diff)
{
OstreeDiffFlags diff_flags = OSTREE_DIFF_FLAGS_NONE;
if (opt_no_xattrs)
diff_flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS;
if (!parse_file_or_commit (repo, src, &srcf, cancellable, error))
goto out;
if (!parse_file_or_commit (repo, target, &targetf, cancellable, error))
@ -171,7 +178,7 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **
removed = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
added = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, srcf, targetf, modified, removed, added, cancellable, error))
if (!ostree_diff_dirs (diff_flags, srcf, targetf, modified, removed, added, cancellable, error))
goto out;
ostree_diff_print (srcf, targetf, modified, removed, added);

148
src/ostree/ot-builtin-export.c Normal file
View File

@ -0,0 +1,148 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2016 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#include "config.h"
#include "ot-main.h"
#include "ot-builtins.h"
#include "ostree.h"
#include "ostree-repo-file.h"
#include "otutil.h"
#ifdef HAVE_LIBARCHIVE
#include <archive.h>
#include <archive_entry.h>
#endif
static char *opt_output_path;
static gboolean opt_no_xattrs;
static GOptionEntry options[] = {
{ "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL },
{ "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" },
{ NULL }
};
#ifdef HAVE_LIBARCHIVE
static void
propagate_libarchive_error (GError **error,
struct archive *a)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"%s", archive_error_string (a));
}
#endif
gboolean
ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error)
{
GOptionContext *context;
glnx_unref_object OstreeRepo *repo = NULL;
gboolean ret = FALSE;
const char *rev;
g_autoptr(GFile) root = NULL;
g_autofree char *commit = NULL;
g_autoptr(GVariant) commit_data = NULL;
struct archive *a;
OstreeRepoArchiveOptions opts = { 0, };
context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format");
if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
goto out;
#ifdef HAVE_LIBARCHIVE
if (argc <= 1)
{
ot_util_usage_error (context, "A COMMIT argument is required", error);
goto out;
}
rev = argv[1];
a = archive_write_new ();
/* Yes, this is hardcoded for now. There is
* archive_write_set_format_filter_by_ext() but it's fairly magic.
* Many programs have support now for GNU tar, so should be a good
* default. I also don't want to lock us into everything libarchive
* supports.
*/
if (archive_write_set_format_gnutar (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
if (archive_write_add_filter_none (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
if (opt_output_path)
{
if (archive_write_open_filename (a, opt_output_path) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
}
else
{
if (archive_write_open_FILE (a, stdout) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
}
if (opt_no_xattrs)
opts.disable_xattrs = TRUE;
if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error))
goto out;
if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_data, error))
goto out;
opts.timestamp_secs = ostree_commit_get_timestamp (commit_data);
if (!ostree_repo_write_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a,
cancellable, error))
goto out;
if (archive_write_close (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
#else
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree is not compiled with libarchive support");
goto out;
#endif
ret = TRUE;
out:
if (context)
g_option_context_free (context);
return ret;
}

1
src/ostree/ot-builtins.h
View File

@ -35,6 +35,7 @@ BUILTINPROTO(checkout);
BUILTINPROTO(checksum);
BUILTINPROTO(commit);
BUILTINPROTO(diff);
BUILTINPROTO(export);
BUILTINPROTO(gpg_sign);
BUILTINPROTO(init);
BUILTINPROTO(log);

50
tests/test-export.sh Executable file
View File

@ -0,0 +1,50 @@
#!/bin/bash
#
# Copyright (C) 2016 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -euo pipefail
. $(dirname $0)/libtest.sh
setup_test_repository "archive-z2"
echo '1..2'
$OSTREE checkout test2 test2-co
$OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co
rm test2-co -rf
cd ${test_tmpdir}
${OSTREE} 'export' test2-noxattrs -o test2.tar
mkdir t
(cd t && tar xf ../test2.tar)
ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt
assert_file_empty diff.txt
rm test2.tar diff.txt t -rf
echo 'ok export gnutar diff (no xattrs)'
cd ${test_tmpdir}
${OSTREE} 'export' test2 -o test2.tar
${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar
ostree --repo=repo diff test2 test2-from-tar
assert_file_empty diff.txt
rm test2.tar diff.txt t -rf
echo 'ok export import'