From 3742c329459615eca34fdd374673c76ac648ffba Mon Sep 17 00:00:00 2001 From: Anne LoVerso Date: Tue, 5 Aug 2014 08:41:50 -0400 Subject: [PATCH] repo-pull: Allow pulling only one directory Changes the pull API to allow pulling only a single directory instead of the whole deployment. This option is utilized by the check-diff option in rpm-ostree. Add a new state directory to hold .commitpartial files, so we know that we've only downloaded partial state. --- Makefile-tests.am | 1 + src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo-prune.c | 23 ++++ src/libostree/ostree-repo-pull.c | 136 ++++++++++++++++++++++-- src/libostree/ostree-repo.c | 7 ++ src/libostree/ostree-repo.h | 10 ++ src/libostree/ostree-sysroot-upgrader.c | 23 +++- src/libostree/ostree-sysroot-upgrader.h | 9 ++ src/ostree/ot-builtin-pull.c | 19 ++-- tests/test-pull-subpath.sh | 51 +++++++++ 10 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 tests/test-pull-subpath.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 5c868410..c36b1b84 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -23,6 +23,7 @@ insttest_PROGRAMS = insttestdir=$(pkglibexecdir)/installed-tests testfiles = test-basic \ + test-pull-subpath \ test-archivez \ test-remote-add \ test-commit-sign \ diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 379460ab..34f6cde1 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -40,6 +40,7 @@ struct OstreeRepo { GFile *local_heads_dir; GFile *remote_heads_dir; GFile *objects_dir; + GFile *state_dir; int objects_dir_fd; GFile *deltas_dir; GFile *uncompressed_objects_dir; diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index ac5364ac..05ba43a1 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -35,6 +35,23 @@ typedef struct { guint64 freed_bytes; } OtPruneData; +static gboolean +prune_commitpartial_file (OstreeRepo *repo, + const char *checksum, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gs_unref_object GFile *objpath = ot_gfile_resolve_path_printf (repo->repodir, "state/%s.commitpartial", checksum); + + if (!ot_gfile_ensure_unlinked (objpath, cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +} + static gboolean maybe_prune_loose_object (OtPruneData *data, OstreeRepoPruneFlags flags, @@ -54,6 +71,12 @@ maybe_prune_loose_object (OtPruneData *data, { guint64 storage_size = 0; + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + { + if (!prune_commitpartial_file (data->repo, checksum, cancellable, error)) + goto out; + } + if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, &storage_size, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 76798d6e..7c85ccd6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -66,7 +66,10 @@ typedef struct { guint n_fetched_content; guint64 start_time; - + + char *dir; + gboolean commitpartial_exists; + gboolean have_previous_bytes; guint64 previous_bytes_sec; guint64 previous_total_downloaded; @@ -383,6 +386,8 @@ scan_dirtree_object (OtPullData *pull_data, gs_unref_variant GVariant *tree = NULL; gs_unref_variant GVariant *files_variant = NULL; gs_unref_variant GVariant *dirs_variant = NULL; + char *subdir_target = NULL; + const char *dirname = NULL; if (recursion_depth > OSTREE_MAX_RECURSION) { @@ -398,8 +403,13 @@ scan_dirtree_object (OtPullData *pull_data, /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ files_variant = g_variant_get_child_value (tree, 0); dirs_variant = g_variant_get_child_value (tree, 1); - - n = g_variant_n_children (files_variant); + + /* Skip files if we're traversing a request only directory */ + if (pull_data->dir) + n = 0; + else + n = g_variant_n_children (files_variant); + for (i = 0; i < n; i++) { const char *filename; @@ -425,11 +435,34 @@ scan_dirtree_object (OtPullData *pull_data, file_checksum = NULL; /* Transfer ownership */ } } - + + + if (pull_data->dir) + { + const char *subpath = NULL; + const char *nextslash = NULL; + g_assert (pull_data->dir[0] == '/'); // assert it starts with / like "/usr/share/rpm" + subpath = pull_data->dir + 1; // refers to name minus / like "usr/share/rpm" + nextslash = strchr (subpath, '/'); //refers to start of next slash like "/share/rpm" + + if (nextslash) + { + subdir_target = g_strndup (subpath, nextslash - subpath); // refers to first dir, like "usr" + g_free (pull_data->dir); + pull_data->dir = g_strdup (nextslash); // sets dir to new deeper level like "/share/rpm" + } + else // we're as deep as it goes, i.e. subpath = "rpm" + { + subdir_target = g_strdup (subpath); + g_clear_pointer (&pull_data->dir, g_free); + pull_data->dir = NULL; + } + } + n = g_variant_n_children (dirs_variant); + for (i = 0; i < n; i++) { - const char *dirname; gs_unref_variant GVariant *tree_csum = NULL; gs_unref_variant GVariant *meta_csum = NULL; @@ -439,6 +472,9 @@ scan_dirtree_object (OtPullData *pull_data, if (!ot_util_filename_validate (dirname, error)) goto out; + if (subdir_target && strcmp (subdir_target, dirname) != 0) + continue; + if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum), OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1, cancellable, error)) @@ -631,6 +667,15 @@ on_metadata_writed (GObject *object, check_outstanding_requests_handle_error (pull_data, local_error); } +/* GFile pointing to the /state/.commitpartial file */ +static GFile * +get_commitpartial_path (OstreeRepo *repo, + const char *commit) +{ + gs_free char *commitpartial_filename = g_strdup_printf ("%s.commitpartial", commit); + return g_file_get_child (repo->state_dir, commitpartial_filename); +} + static void meta_fetch_on_complete (GObject *object, GAsyncResult *result, @@ -685,6 +730,20 @@ meta_fetch_on_complete (GObject *object, goto out; (void) gs_file_unlink (temp_path, NULL, NULL); + + /* Write the commitpartial file now while we're still fetching data */ + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + { + GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, checksum); + + if (!g_file_query_exists (commitpartial_path, NULL)) + { + if (!g_file_replace_contents (commitpartial_path, "", 0, NULL, FALSE, + G_FILE_CREATE_REPLACE_DESTINATION, NULL, + pull_data->cancellable, error)) + goto out; + } + } ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata, pull_data->cancellable, @@ -814,7 +873,21 @@ scan_one_metadata_object_c (OtPullData *pull_data, } else if (is_stored) { - if (pull_data->transaction_resuming || is_requested) + gboolean do_scan = pull_data->transaction_resuming || is_requested || pull_data->commitpartial_exists; + + /* For commits, check whether we only had a partial fetch */ + if (!do_scan && objtype == OSTREE_OBJECT_TYPE_COMMIT) + { + gs_unref_object GFile *commitpartial_file = get_commitpartial_path (pull_data->repo, tmp_checksum); + + if (g_file_query_exists (commitpartial_file, NULL)) + { + do_scan = TRUE; + pull_data->commitpartial_exists = TRUE; + } + } + + if (do_scan) { switch (objtype) { @@ -1024,6 +1097,24 @@ ostree_repo_pull (OstreeRepo *self, OstreeAsyncProgress *progress, GCancellable *cancellable, GError **error) +{ + return ostree_repo_pull_one_dir (self, remote_name, NULL, refs_to_fetch, flags, progress, cancellable, error); +} + +/** + * ostree_repo_pull_one_dir: + * + * Like ostree_repo_pull(), but supports pulling only a subpath. + */ +gboolean +ostree_repo_pull_one_dir (OstreeRepo *self, + const char *remote_name, + const char *dir_to_pull, + char **refs_to_fetch, + OstreeRepoPullFlags flags, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; GHashTableIter hash_iter; @@ -1045,6 +1136,9 @@ ostree_repo_pull (OstreeRepo *self, guint64 bytes_transferred; guint64 end_time; + if (dir_to_pull) + g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); + pull_data->async_error = error; pull_data->main_context = g_main_context_ref_thread_default (); pull_data->loop = g_main_loop_new (pull_data->main_context, FALSE); @@ -1059,6 +1153,7 @@ ostree_repo_pull (OstreeRepo *self, (GDestroyNotify)g_free, NULL); pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); + pull_data->dir = g_strdup (dir_to_pull); pull_data->start_time = g_get_monotonic_time (); @@ -1209,7 +1304,7 @@ ostree_repo_pull (OstreeRepo *self, { if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) goto out; - + /* Transfer ownership of contents */ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); } @@ -1241,6 +1336,12 @@ ostree_repo_pull (OstreeRepo *self, } } + /* Create the state directory here - it's new with the commitpartial code, + * and may not exist in older repositories. + */ + if (!gs_file_ensure_directory (pull_data->repo->state_dir, FALSE, pull_data->cancellable, error)) + goto out; + pull_data->phase = OSTREE_PULL_PHASE_FETCHING_OBJECTS; if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming, @@ -1328,6 +1429,27 @@ ostree_repo_pull (OstreeRepo *self, ostree_async_progress_set_status (pull_data->progress, msg); } + /* iterate over commits fetched and delete any commitpartial files */ + if (!dir_to_pull) + { + g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *checksum = value; + gs_unref_object GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, checksum); + if (!ot_gfile_ensure_unlinked (commitpartial_path, cancellable, error)) + goto out; + } + g_hash_table_iter_init (&hash_iter, commits_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *commit = value; + gs_unref_object GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, commit); + if (!ot_gfile_ensure_unlinked (commitpartial_path, cancellable, error)) + goto out; + } + } + ret = TRUE; out: if (pull_data->main_context) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 6f04360f..8a85ef25 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -100,6 +100,7 @@ ostree_repo_finalize (GObject *object) if (self->objects_dir_fd != -1) (void) close (self->objects_dir_fd); g_clear_object (&self->deltas_dir); + g_clear_object (&self->state_dir); g_clear_object (&self->uncompressed_objects_dir); if (self->uncompressed_objects_dir_fd != -1) (void) close (self->uncompressed_objects_dir_fd); @@ -178,6 +179,7 @@ ostree_repo_constructed (GObject *object) self->uncompressed_objects_dir = g_file_resolve_relative_path (self->repodir, "uncompressed-objects-cache/objects"); self->deltas_dir = g_file_get_child (self->repodir, "deltas"); self->uncompressed_objects_dir = g_file_get_child (self->repodir, "uncompressed-objects-cache"); + self->state_dir = g_file_get_child (self->repodir, "state"); self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache"); self->config_file = g_file_get_child (self->repodir, "config"); @@ -586,6 +588,11 @@ ostree_repo_create (OstreeRepo *self, if (!g_file_make_directory (grandchild, cancellable, error)) goto out; + g_clear_object (&child); + child = g_file_get_child (self->repodir, "state"); + if (!g_file_make_directory (child, cancellable, error)) + goto out; + if (!ostree_repo_open (self, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 26fd6cc1..cace5e5b 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -538,6 +538,16 @@ gboolean ostree_repo_pull (OstreeRepo *self, GCancellable *cancellable, GError **error); +gboolean +ostree_repo_pull_one_dir (OstreeRepo *self, + const char *remote_name, + const char *dir_to_pull, + char **refs_to_fetch, + OstreeRepoPullFlags flags, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error); + gboolean ostree_repo_sign_commit (OstreeRepo *self, const gchar *commit_checksum, const gchar *key_id, diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 61b6309a..10cddea0 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -427,6 +427,27 @@ ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader *self, gboolean *out_changed, GCancellable *cancellable, GError **error) +{ + return ostree_sysroot_upgrader_pull_one_dir (self, NULL, flags, upgrader_flags, progress, out_changed, cancellable, error); +} + +/** + * ostree_sysroot_upgrader_pull_one_dir: + * + * Like ostree_sysroot_upgrader_pull(), but allows retrieving just a + * subpath of the tree. This can be used to download metadata files + * from inside the tree such as package databases. + * + */ +gboolean +ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, + const char *dir_to_pull, + OstreeRepoPullFlags flags, + OstreeSysrootUpgraderPullFlags upgrader_flags, + OstreeAsyncProgress *progress, + gboolean *out_changed, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; gs_unref_object OstreeRepo *repo = NULL; @@ -448,7 +469,7 @@ ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader *self, if (self->origin_remote) { - if (!ostree_repo_pull (repo, self->origin_remote, refs_to_fetch, + if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch, flags, progress, cancellable, error)) goto out; diff --git a/src/libostree/ostree-sysroot-upgrader.h b/src/libostree/ostree-sysroot-upgrader.h index a4e6b77a..cf601248 100644 --- a/src/libostree/ostree-sysroot-upgrader.h +++ b/src/libostree/ostree-sysroot-upgrader.h @@ -65,6 +65,15 @@ gboolean ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error); +gboolean ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, + const char *dir_to_pull, + OstreeRepoPullFlags flags, + OstreeSysrootUpgraderPullFlags upgrader_flags, + OstreeAsyncProgress *progress, + gboolean *out_changed, + GCancellable *cancellable, + GError **error); + gboolean ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 1cf3bb78..31feef47 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -29,12 +29,14 @@ static gboolean opt_disable_fsync; static gboolean opt_mirror; - -static GOptionEntry options[] = { - { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, - { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, - { NULL } -}; +static char* opt_subpath; + + static GOptionEntry options[] = { + { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, + { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, + { NULL } + }; gboolean ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error) @@ -95,8 +97,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *canc progress = ostree_async_progress_new_and_connect (ot_common_pull_progress, console); } - if (!ostree_repo_pull (repo, remote, refs_to_fetch ? (char**)refs_to_fetch->pdata : NULL, - pullflags, progress, cancellable, error)) + if (!ostree_repo_pull_one_dir (repo, remote, opt_subpath, + refs_to_fetch ? (char**)refs_to_fetch->pdata : NULL, + pullflags, progress, cancellable, error)) goto out; if (progress) diff --git a/tests/test-pull-subpath.sh b/tests/test-pull-subpath.sh new file mode 100644 index 00000000..7e1bca5b --- /dev/null +++ b/tests/test-pull-subpath.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (C) 2014 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -e + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive-z2" + +echo '1..1' + +echo "SUBDIR TEST" + +repopath=${test_tmpdir}/ostree-srv/gnomerepo +cp -a ${repopath} ${repopath}.orig + +cd ${test_tmpdir} +rm repo -rf +mkdir repo +${CMD_PREFIX} ostree --repo=repo init +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + +${CMD_PREFIX} ostree --repo=repo pull --subpath=/baz origin main + +${CMD_PREFIX} ostree --repo=repo ls origin:main /baz +if ${CMD_PREFIX} ostree --repo=repo ls origin:main /firstfile 2>err.txt; then + assert_not_reached +fi +assert_file_has_content err.txt "Couldn't find file object" +rev=$(ostree --repo=repo rev-parse origin:main) +assert_has_file repo/state/${rev}.commitpartial + +${CMD_PREFIX} ostree --repo=repo pull origin main +assert_not_has_file repo/state/${rev}.commitpartial +${CMD_PREFIX} ostree --repo=repo fsck