create-usb: Add a create-usb command to complement OstreeRepoFinderMount

This can be used to put OSTree repositories on USB sticks in a format
recognised by OstreeRepoFinderMount.

Signed-off-by: Philip Withnall <withnall@endlessm.com>

Closes: #1182
Approved by: cgwalters
This commit is contained in:
Philip Withnall 2017-09-15 16:05:12 +01:00 committed by Atomic Bot
parent f923c2e1ea
commit 9546e6795e
9 changed files with 532 additions and 1 deletions

View File

@ -54,7 +54,10 @@ ostree_SOURCES = src/ostree/main.c \
$(NULL)
if ENABLE_EXPERIMENTAL_API
ostree_SOURCES += src/ostree/ot-builtin-find-remotes.c
ostree_SOURCES += \
src/ostree/ot-builtin-create-usb.c \
src/ostree/ot-builtin-find-remotes.c \
$(NULL)
endif
# Admin subcommand

View File

@ -115,6 +115,7 @@ _installed_or_uninstalled_test_scripts = \
$(NULL)
experimental_test_scripts = \
tests/test-create-usb.sh \
tests/test-find-remotes.sh \
tests/test-fsck-collections.sh \
tests/test-init-collections.sh \
@ -124,9 +125,15 @@ experimental_test_scripts = \
tests/test-summary-collections.sh \
tests/test-pull-collections.sh \
$(NULL)
test_extra_programs = $(NULL)
tests_repo_finder_mount_SOURCES = tests/repo-finder-mount.c
tests_repo_finder_mount_CFLAGS = $(common_tests_cflags)
tests_repo_finder_mount_LDADD = $(common_tests_ldadd) libostreetest.la
if ENABLE_EXPERIMENTAL_API
_installed_or_uninstalled_test_scripts += $(experimental_test_scripts)
test_extra_programs += tests/repo-finder-mount
else
EXTRA_DIST += $(experimental_test_scripts)
endif

View File

@ -42,6 +42,7 @@ static OstreeCommand commands[] = {
{ "export", ostree_builtin_export },
#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
{ "find-remotes", ostree_builtin_find_remotes },
{ "create-usb", ostree_builtin_create_usb },
#endif
{ "fsck", ostree_builtin_fsck },
{ "gpg-sign", ostree_builtin_gpg_sign },

View File

@ -0,0 +1,276 @@
/*
* Copyright © 2017 Endless Mobile, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Authors:
* - Philip Withnall <withnall@endlessm.com>
*/
#include "config.h"
#include "ot-main.h"
#include "ot-builtins.h"
#include "ostree.h"
#include "otutil.h"
#include "ostree-remote-private.h"
static gboolean opt_disable_fsync = FALSE;
static char *opt_destination_repo = NULL;
static GOptionEntry options[] =
{
{ "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL },
{ "destination-repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_destination_repo, "Use custom repository directory within the mount", NULL },
{ NULL }
};
/* TODO: Add a man page. */
gboolean
ostree_builtin_create_usb (int argc,
char **argv,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = NULL;
g_autoptr(OstreeAsyncProgress) progress = NULL;
g_auto(GLnxConsoleRef) console = { 0, };
context = g_option_context_new ("MOUNT-PATH COLLECTION-ID REF [COLLECTION-ID REF...] - Copy the refs to a USB stick");
/* Parse options. */
g_autoptr(OstreeRepo) src_repo = NULL;
if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &src_repo, cancellable, error))
return FALSE;
if (argc < 2)
{
ot_util_usage_error (context, "A MOUNT-PATH must be specified", error);
return FALSE;
}
if (argc < 4)
{
ot_util_usage_error (context, "At least one COLLECTION-ID REF pair must be specified", error);
return FALSE;
}
if (argc % 2 == 1)
{
ot_util_usage_error (context, "Only complete COLLECTION-ID REF pairs may be specified", error);
return FALSE;
}
/* Open the USB stick, which must exist. Allow automounting and following symlinks. */
const char *mount_root_path = argv[1];
struct stat mount_root_stbuf;
glnx_fd_close int mount_root_dfd = -1;
if (!glnx_opendirat (AT_FDCWD, mount_root_path, TRUE, &mount_root_dfd, error))
return FALSE;
if (!glnx_fstat (mount_root_dfd, &mount_root_stbuf, error))
return FALSE;
/* Read in the refs to add to the USB stick. */
g_autoptr(GPtrArray) refs = g_ptr_array_new_full (argc, (GDestroyNotify) ostree_collection_ref_free);
for (gsize i = 2; i < argc; i += 2)
{
if (!ostree_validate_collection_id (argv[i], error) ||
!ostree_validate_rev (argv[i + 1], error))
return FALSE;
g_ptr_array_add (refs, ostree_collection_ref_new (argv[i], argv[i + 1]));
}
/* Open the destination repository on the USB stick or create it if it doesnt exist.
* Check its below @mount_root_path, and that its not the same as the source
* repository.
*
* If the destination file system supports xattrs (for example, ext4), we use
* a BARE_USER repository; if it doesnt (for example, FAT), we use ARCHIVE.
* In either case, we want a lossless repository. */
const char *dest_repo_path = (opt_destination_repo != NULL) ? opt_destination_repo : ".ostree/repo";
if (!glnx_shutil_mkdir_p_at (mount_root_dfd, dest_repo_path, 0755, cancellable, error))
return FALSE;
OstreeRepoMode mode = OSTREE_REPO_MODE_BARE_USER;
if (TEMP_FAILURE_RETRY (fgetxattr (mount_root_dfd, "user.test", NULL, 0)) < 0 &&
errno == ENOTSUP)
mode = OSTREE_REPO_MODE_ARCHIVE;
g_debug ("%s: Creating repository in mode %u", G_STRFUNC, mode);
g_autoptr(OstreeRepo) dest_repo = ostree_repo_create_at (mount_root_dfd, dest_repo_path,
mode, NULL, cancellable, error);
if (dest_repo == NULL)
return FALSE;
struct stat dest_repo_stbuf;
if (!glnx_fstat (ostree_repo_get_dfd (dest_repo), &dest_repo_stbuf, error))
return FALSE;
if (dest_repo_stbuf.st_dev != mount_root_stbuf.st_dev)
{
ot_util_usage_error (context, "--destination-repo must be a descendent of MOUNT-PATH", error);
return FALSE;
}
if (ostree_repo_equal (src_repo, dest_repo))
{
ot_util_usage_error (context, "--destination-repo must not be the source repository", error);
return FALSE;
}
if (!ostree_ensure_repo_writable (dest_repo, error))
return FALSE;
if (opt_disable_fsync)
ostree_repo_set_disable_fsync (dest_repo, TRUE);
/* Copy across all of the collectionrefs to the destination repo. */
GVariantBuilder refs_builder;
g_variant_builder_init (&refs_builder, G_VARIANT_TYPE ("a(sss)"));
for (gsize i = 0; i < refs->len; i++)
{
const OstreeCollectionRef *ref = g_ptr_array_index (refs, i);
g_variant_builder_add (&refs_builder, "(sss)",
ref->collection_id, ref->ref_name, "");
}
{
GVariantBuilder builder;
g_autoptr(GVariant) opts = NULL;
OstreeRepoPullFlags flags = OSTREE_REPO_PULL_FLAGS_MIRROR;
glnx_console_lock (&console);
if (console.is_tty)
progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console);
g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}"));
g_variant_builder_add (&builder, "{s@v}", "collection-refs",
g_variant_new_variant (g_variant_builder_end (&refs_builder)));
g_variant_builder_add (&builder, "{s@v}", "flags",
g_variant_new_variant (g_variant_new_int32 (flags)));
g_variant_builder_add (&builder, "{s@v}", "depth",
g_variant_new_variant (g_variant_new_int32 (0)));
opts = g_variant_ref_sink (g_variant_builder_end (&builder));
g_autofree char *src_repo_uri = g_file_get_uri (ostree_repo_get_path (src_repo));
if (!ostree_repo_pull_with_options (dest_repo, src_repo_uri,
opts,
progress,
cancellable, error))
{
ostree_repo_abort_transaction (dest_repo, cancellable, NULL);
return FALSE;
}
if (progress != NULL)
ostree_async_progress_finish (progress);
}
/* Ensure a summary file is present to make it easier to look up commit checksums. */
/* FIXME: It should be possible to work without this, but find_remotes_cb() in
* ostree-repo-pull.c currently assumes a summary file (signed or unsigned) is
* present. */
struct stat stbuf;
if (!glnx_fstatat_allow_noent (ostree_repo_get_dfd (dest_repo), "summary", &stbuf, 0, error))
return FALSE;
if (errno == ENOENT &&
!ostree_repo_regenerate_summary (dest_repo, NULL, cancellable, error))
return FALSE;
/* Add the symlinks .ostree/repos.d/@symlink_name → @dest_repo_path, unless
* the @dest_repo_path is a well-known one like ostree/repo, in which case no
* symlink is necessary; #OstreeRepoFinderMount always looks there. */
if (!g_str_equal (dest_repo_path, "ostree/repo") &&
!g_str_equal (dest_repo_path, ".ostree/repo"))
{
if (!glnx_shutil_mkdir_p_at (mount_root_dfd, ".ostree/repos.d", 0755, cancellable, error))
return FALSE;
/* Find a unique name for the symlink. If a symlink already targets
* @dest_repo_path, use that and dont create a new one. */
GLnxDirFdIterator repos_iter;
gboolean need_symlink = TRUE;
if (!glnx_dirfd_iterator_init_at (mount_root_dfd, ".ostree/repos.d", TRUE, &repos_iter, error))
return FALSE;
while (TRUE)
{
struct dirent *repo_dent;
if (!glnx_dirfd_iterator_next_dent (&repos_iter, &repo_dent, cancellable, error))
return FALSE;
if (repo_dent == NULL)
break;
/* Does the symlink already point to this repository? (Or is the
* repository itself present in repos.d?) We already guarantee that
* theyre on the same device. */
if (repo_dent->d_ino == dest_repo_stbuf.st_ino)
{
need_symlink = FALSE;
break;
}
}
/* If we need a symlink, find a unique name for it and create it. */
if (need_symlink)
{
/* Relative to .ostree/repos.d. */
g_autofree char *relative_dest_repo_path = g_build_filename ("..", "..", dest_repo_path, NULL);
guint i;
const guint max_attempts = 100;
for (i = 0; i < max_attempts; i++)
{
g_autofree char *symlink_path = g_strdup_printf (".ostree/repos.d/%02u-generated", i);
int ret = TEMP_FAILURE_RETRY (symlinkat (relative_dest_repo_path, mount_root_dfd, symlink_path));
if (ret < 0 && errno != EEXIST)
return glnx_throw_errno_prefix (error, "symlinkat(%s → %s)", symlink_path, relative_dest_repo_path);
else if (ret >= 0)
break;
}
if (i == max_attempts)
return glnx_throw (error, "Could not find an unused symlink name for the repository");
}
}
/* Report success to the user. */
g_autofree char *src_repo_path = g_file_get_path (ostree_repo_get_path (src_repo));
g_print ("Copied %u/%u refs successfully from %s to %s repository in %s.\n", refs->len, refs->len,
src_repo_path, dest_repo_path, mount_root_path);
return TRUE;
}

View File

@ -39,6 +39,7 @@ BUILTINPROTO(diff);
BUILTINPROTO(export);
#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
BUILTINPROTO(find_remotes);
BUILTINPROTO(create_usb);
#endif
BUILTINPROTO(gpg_sign);
BUILTINPROTO(init);

1
tests/.gitignore vendored
View File

@ -3,6 +3,7 @@
*.test
*.trs
ostree-http-server
repo-finder-mount
run-apache
tmpdir-lifecycle
test-rollsum

View File

@ -603,6 +603,12 @@ skip_without_fuse () {
[ -e /etc/mtab ] || skip "no /etc/mtab"
}
skip_without_experimental () {
if ! ostree --version | grep -q -e '- experimental'; then
skip "No experimental API is compiled in"
fi
}
has_gpgme () {
${CMD_PREFIX} ostree --version > version.txt
assert_file_has_content version.txt '- gpgme'

126
tests/repo-finder-mount.c Normal file
View File

@ -0,0 +1,126 @@
/*
* Copyright © 2017 Endless Mobile, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Authors:
* - Philip Withnall <withnall@endlessm.com>
*/
#include "config.h"
#include <gio/gio.h>
#include <glib.h>
#include <glib-object.h>
#include <locale.h>
#include "ostree-autocleanups.h"
#include "ostree-remote-private.h"
#include "ostree-repo-finder.h"
#include "ostree-repo-finder-mount.h"
#include "ostree-types.h"
#include "test-mock-gio.h"
static void
result_cb (GObject *source_object,
GAsyncResult *result,
gpointer user_data)
{
GAsyncResult **result_out = user_data;
*result_out = g_object_ref (result);
}
static void
collection_ref_free0 (OstreeCollectionRef *ref)
{
g_clear_pointer (&ref, (GDestroyNotify) ostree_collection_ref_free);
}
int
main (int argc, char **argv)
{
g_autoptr(GError) error = NULL;
setlocale (LC_ALL, "");
if (argc < 5 || (argc % 2) != 1)
{
g_printerr ("Usage: %s REPO MOUNT-ROOT COLLECTION-ID REF-NAME [COLLECTION-ID REF-NAME …]\n", argv[0]);
return 1;
}
g_autoptr(GMainContext) context = g_main_context_new ();
g_main_context_push_thread_default (context);
g_autoptr(OstreeRepo) parent_repo = ostree_repo_open_at (AT_FDCWD, argv[1], NULL, &error);
g_assert_no_error (error);
/* Set up a mock volume. */
g_autoptr(GFile) mount_root = g_file_new_for_commandline_arg (argv[2]);
g_autoptr(GMount) mount = G_MOUNT (ostree_mock_mount_new ("mount", mount_root));
g_autoptr(GList) mounts = g_list_prepend (NULL, mount);
g_autoptr(GVolumeMonitor) monitor = ostree_mock_volume_monitor_new (mounts, NULL);
g_autoptr(OstreeRepoFinderMount) finder = ostree_repo_finder_mount_new (monitor);
/* Resolve the refs. */
g_autoptr(GPtrArray) refs = g_ptr_array_new_with_free_func ((GDestroyNotify) collection_ref_free0);
for (gsize i = 3; i < argc; i += 2)
{
const char *collection_id = argv[i];
const char *ref_name = argv[i + 1];
g_ptr_array_add (refs, ostree_collection_ref_new (collection_id, ref_name));
}
g_ptr_array_add (refs, NULL); /* NULL terminator */
g_autoptr(GAsyncResult) result = NULL;
ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder),
(const OstreeCollectionRef * const *) refs->pdata,
parent_repo, NULL, result_cb, &result);
while (result == NULL)
g_main_context_iteration (context, TRUE);
g_autoptr(GPtrArray) results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder),
result, &error);
g_assert_no_error (error);
/* Check that the results are correct: the invalid refs should have been
* ignored, and the valid results canonicalised and deduplicated. */
for (gsize i = 0; i < results->len; i++)
{
const OstreeRepoFinderResult *result = g_ptr_array_index (results, i);
GHashTableIter iter;
OstreeCollectionRef *ref;
const gchar *checksum;
g_hash_table_iter_init (&iter, result->ref_to_checksum);
while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum))
g_print ("%" G_GSIZE_FORMAT " %s %s %s %s\n",
i, ostree_remote_get_name (result->remote),
ref->collection_id, ref->ref_name,
checksum);
}
g_main_context_pop_thread_default (context);
return 0;
}

110
tests/test-create-usb.sh Executable file
View File

@ -0,0 +1,110 @@
#!/bin/bash
#
# Copyright © 2017 Endless Mobile, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#
# Authors:
# - Philip Withnall <withnall@endlessm.com>
set -euo pipefail
. $(dirname $0)/libtest.sh
echo "1..5"
cd ${test_tmpdir}
mkdir repo
ostree_repo_init repo --collection-id org.example.Collection1
mkdir -p tree/root
touch tree/root/a
# Add a few commits
seq 5 | while read i; do
echo a >> tree/root/a
${CMD_PREFIX} ostree --repo=repo commit --branch=test-$i -m test -s test --gpg-homedir="${TEST_GPG_KEYHOME}" --gpg-sign="${TEST_GPG_KEYID_1}" tree
done
${CMD_PREFIX} ostree --repo=repo summary --update --gpg-homedir="${TEST_GPG_KEYHOME}" --gpg-sign="${TEST_GPG_KEYID_1}"
# Pull into a local repository, to more accurately represent the situation of
# creating a USB stick from your local machine.
mkdir local-repo
${CMD_PREFIX} ostree --repo=local-repo init
${CMD_PREFIX} ostree --repo=local-repo remote add remote1 file://$(pwd)/repo --collection-id org.example.Collection1 --gpg-import="${test_tmpdir}/gpghome/key1.asc"
${CMD_PREFIX} ostree --repo=local-repo pull remote1 test-1 test-2 test-3 test-4 test-5
# Simple test to put two refs onto a USB stick.
mkdir dest-mount1
${CMD_PREFIX} ostree --repo=local-repo create-usb dest-mount1 org.example.Collection1 test-1 org.example.Collection1 test-2
assert_has_dir dest-mount1/.ostree/repo
${CMD_PREFIX} ostree --repo=dest-mount1/.ostree/repo refs --collections > dest-refs
assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$"
assert_file_has_content dest-refs "^(org.example.Collection1, test-2)$"
assert_not_file_has_content dest-refs "^(org.example.Collection1, test-3)$"
assert_has_file dest-mount1/.ostree/repo/summary
echo "ok 1 simple usb"
# Test that the repository can be placed in another standard location on the USB stick.
for dest in ostree/repo .ostree/repo; do
rm -rf dest-mount2
mkdir dest-mount2
${CMD_PREFIX} ostree --repo=local-repo create-usb --destination-repo "$dest" dest-mount2 org.example.Collection1 test-1
assert_has_dir "dest-mount2/$dest"
if [ -d dest-mount2/.ostree/repos.d ]; then
ls dest-mount2/.ostree/repos.d | wc -l > repo-links
assert_file_has_content repo-links "^0$"
fi
done
echo "ok 2 usb in standard location"
# Test that the repository can be placed in a non-standard location and gets a symlink to it.
mkdir dest-mount3
${CMD_PREFIX} ostree --repo=local-repo create-usb --destination-repo some-dest dest-mount3 org.example.Collection1 test-1
assert_has_dir "dest-mount3/some-dest"
assert_symlink_has_content "dest-mount3/.ostree/repos.d/00-generated" "/some-dest$"
${CMD_PREFIX} ostree --repo=dest-mount3/.ostree/repos.d/00-generated refs --collections > dest-refs
assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$"
assert_has_file dest-mount3/.ostree/repos.d/00-generated/summary
echo "ok 3 usb in non-standard location"
# Test that adding an additional ref to an existing USB repository works.
${CMD_PREFIX} ostree --repo=local-repo create-usb --destination-repo some-dest dest-mount3 org.example.Collection1 test-2 org.example.Collection1 test-3
assert_has_dir "dest-mount3/some-dest"
assert_symlink_has_content "dest-mount3/.ostree/repos.d/00-generated" "/some-dest$"
${CMD_PREFIX} ostree --repo=dest-mount3/.ostree/repos.d/00-generated refs --collections > dest-refs
assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$"
assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$"
assert_file_has_content dest-refs "^(org.example.Collection1, test-3)$"
assert_has_file dest-mount3/.ostree/repos.d/00-generated/summary
echo "ok 4 adding ref to an existing usb"
# Check that #OstreeRepoFinderMount works from a volume initialised uing create-usb.
mkdir finder-repo
ostree_repo_init finder-repo
${CMD_PREFIX} ostree --repo=finder-repo remote add remote1 file://$(pwd)/just-needed-for-the-keyring --collection-id org.example.Collection1 --gpg-import="${test_tmpdir}/gpghome/key1.asc"
${test_builddir}/repo-finder-mount finder-repo dest-mount1 org.example.Collection1 test-1 org.example.Collection1 test-2 &> out
assert_file_has_content out "^0 .*_2Fdest-mount1_2F.ostree_2Frepo_remote1.trustedkeys.gpg org.example.Collection1 test-1 $(ostree --repo=repo show test-1)$"
assert_file_has_content out "^0 .*_2Fdest-mount1_2F.ostree_2Frepo_remote1.trustedkeys.gpg org.example.Collection1 test-2 $(ostree --repo=repo show test-2)$"
echo "ok 5 find from usb repo"