repo-pull: Allow pulling only one directory

Changes the pull API to allow pulling only a single directory instead
of the whole deployment.  This option is utilized by the check-diff
option in rpm-ostree.

Add a new state directory to hold <checksum>.commitpartial files, so
we know that we've only downloaded partial state.

Makefile-tests.am

@@ -23,6 +23,7 @@ insttest_PROGRAMS =
 
 insttestdir=$(pkglibexecdir)/installed-tests
 testfiles = test-basic \
+	test-pull-subpath \
 	test-archivez \
 	test-remote-add \
         test-commit-sign \

src/libostree/ostree-repo-private.h

@@ -40,6 +40,7 @@ struct OstreeRepo {
   GFile *local_heads_dir;
   GFile *remote_heads_dir;
   GFile *objects_dir;
+  GFile *state_dir;
   int objects_dir_fd;
   GFile *deltas_dir;
   GFile *uncompressed_objects_dir;

src/libostree/ostree-repo-prune.c

@@ -35,6 +35,23 @@ typedef struct {
   guint64 freed_bytes;
 } OtPruneData;
 
+static gboolean
+prune_commitpartial_file (OstreeRepo    *repo,
+                          const char    *checksum,
+                          GCancellable  *cancellable,
+                          GError       **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_object GFile *objpath = ot_gfile_resolve_path_printf (repo->repodir, "state/%s.commitpartial", checksum);
+  
+  if (!ot_gfile_ensure_unlinked (objpath, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
 static gboolean
 maybe_prune_loose_object (OtPruneData        *data,
                           OstreeRepoPruneFlags    flags,
@@ -54,6 +71,12 @@ maybe_prune_loose_object (OtPruneData        *data,
         {
           guint64 storage_size = 0;
 
+          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+            {
+              if (!prune_commitpartial_file (data->repo, checksum, cancellable, error))
+                goto out;
+            }
+
           if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum,
                                                       &storage_size, cancellable, error))
             goto out;

src/libostree/ostree-repo-pull.c

@@ -66,7 +66,10 @@ typedef struct {
   guint             n_fetched_content;
 
   guint64           start_time;
-  
+
+  char         *dir;
+  gboolean      commitpartial_exists;
+
   gboolean      have_previous_bytes;
   guint64       previous_bytes_sec;
   guint64       previous_total_downloaded;
@@ -383,6 +386,8 @@ scan_dirtree_object (OtPullData   *pull_data,
   gs_unref_variant GVariant *tree = NULL;
   gs_unref_variant GVariant *files_variant = NULL;
   gs_unref_variant GVariant *dirs_variant = NULL;
+  char *subdir_target = NULL;
+  const char *dirname = NULL;
 
   if (recursion_depth > OSTREE_MAX_RECURSION)
     {
@@ -398,8 +403,13 @@ scan_dirtree_object (OtPullData   *pull_data,
   /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
   files_variant = g_variant_get_child_value (tree, 0);
   dirs_variant = g_variant_get_child_value (tree, 1);
-      
+
-  n = g_variant_n_children (files_variant);
+  /* Skip files if we're traversing a request only directory */
+  if (pull_data->dir)
+    n = 0;
+  else
+    n = g_variant_n_children (files_variant);
+
   for (i = 0; i < n; i++)
     {
       const char *filename;
@@ -425,11 +435,34 @@ scan_dirtree_object (OtPullData   *pull_data,
           file_checksum = NULL;  /* Transfer ownership */
         }
     }
-      
+
+
+    if (pull_data->dir)
+      {
+        const char *subpath = NULL;  
+        const char *nextslash = NULL;
+        g_assert (pull_data->dir[0] == '/'); // assert it starts with / like "/usr/share/rpm"
+        subpath = pull_data->dir + 1;  // refers to name minus / like "usr/share/rpm"
+        nextslash = strchr (subpath, '/'); //refers to start of next slash like "/share/rpm"
+
+        if (nextslash)
+          {
+            subdir_target = g_strndup (subpath, nextslash - subpath); // refers to first dir, like "usr"
+            g_free (pull_data->dir);
+            pull_data->dir = g_strdup (nextslash); // sets dir to new deeper level like "/share/rpm"
+          }
+        else // we're as deep as it goes, i.e. subpath = "rpm"
+          {
+            subdir_target = g_strdup (subpath); 
+            g_clear_pointer (&pull_data->dir, g_free);
+            pull_data->dir = NULL;
+          }
+        }
+
   n = g_variant_n_children (dirs_variant);
+
   for (i = 0; i < n; i++)
     {
-      const char *dirname;
       gs_unref_variant GVariant *tree_csum = NULL;
       gs_unref_variant GVariant *meta_csum = NULL;
 
@@ -439,6 +472,9 @@ scan_dirtree_object (OtPullData   *pull_data,
       if (!ot_util_filename_validate (dirname, error))
         goto out;
 
+      if (subdir_target && strcmp (subdir_target, dirname) != 0)
+        continue;
+      
       if (!scan_one_metadata_object_c (pull_data, ostree_checksum_bytes_peek (tree_csum),
                                        OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
                                        cancellable, error))
@@ -631,6 +667,15 @@ on_metadata_writed (GObject           *object,
   check_outstanding_requests_handle_error (pull_data, local_error);
 }
 
+/* GFile pointing to the <repodir>/state/<checksum>.commitpartial file */
+static GFile *
+get_commitpartial_path (OstreeRepo  *repo,
+                        const char  *commit)
+{
+  gs_free char *commitpartial_filename = g_strdup_printf ("%s.commitpartial", commit);
+  return g_file_get_child (repo->state_dir, commitpartial_filename);
+}
+
 static void
 meta_fetch_on_complete (GObject           *object,
                         GAsyncResult      *result,
@@ -685,6 +730,20 @@ meta_fetch_on_complete (GObject           *object,
         goto out;
 
       (void) gs_file_unlink (temp_path, NULL, NULL);
+
+      /* Write the commitpartial file now while we're still fetching data */
+      if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+        {
+          GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, checksum);
+
+          if (!g_file_query_exists (commitpartial_path, NULL))
+            {
+              if (!g_file_replace_contents (commitpartial_path, "", 0, NULL, FALSE, 
+                                            G_FILE_CREATE_REPLACE_DESTINATION, NULL,
+                                            pull_data->cancellable, error))
+                goto out;
+            }
+        }
       
       ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata,
                                         pull_data->cancellable,
@@ -814,7 +873,21 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
     }
   else if (is_stored)
     {
-      if (pull_data->transaction_resuming || is_requested)
+      gboolean do_scan = pull_data->transaction_resuming || is_requested || pull_data->commitpartial_exists;
+
+      /* For commits, check whether we only had a partial fetch */
+      if (!do_scan && objtype == OSTREE_OBJECT_TYPE_COMMIT)
+        {
+          gs_unref_object GFile *commitpartial_file = get_commitpartial_path (pull_data->repo, tmp_checksum);
+
+          if (g_file_query_exists (commitpartial_file, NULL))
+            {
+              do_scan = TRUE;
+              pull_data->commitpartial_exists = TRUE;
+            }
+        }
+
+      if (do_scan)
         {
           switch (objtype)
             {
@@ -1024,6 +1097,24 @@ ostree_repo_pull (OstreeRepo               *self,
                   OstreeAsyncProgress      *progress,
                   GCancellable             *cancellable,
                   GError                  **error)
+{
+  return ostree_repo_pull_one_dir (self, remote_name, NULL, refs_to_fetch, flags, progress, cancellable, error);
+}
+
+/**
+ * ostree_repo_pull_one_dir:
+ *
+ * Like ostree_repo_pull(), but supports pulling only a subpath.
+ */
+gboolean
+ostree_repo_pull_one_dir (OstreeRepo               *self,
+                          const char               *remote_name,
+                          const char               *dir_to_pull,
+                          char                    **refs_to_fetch,
+                          OstreeRepoPullFlags       flags,
+                          OstreeAsyncProgress      *progress,
+                          GCancellable             *cancellable,
+                          GError                  **error)
 {
   gboolean ret = FALSE;
   GHashTableIter hash_iter;
@@ -1045,6 +1136,9 @@ ostree_repo_pull (OstreeRepo               *self,
   guint64 bytes_transferred;
   guint64 end_time;
 
+  if (dir_to_pull)
+    g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
+
   pull_data->async_error = error;
   pull_data->main_context = g_main_context_ref_thread_default ();
   pull_data->loop = g_main_loop_new (pull_data->main_context, FALSE);
@@ -1059,6 +1153,7 @@ ostree_repo_pull (OstreeRepo               *self,
                                                         (GDestroyNotify)g_free, NULL);
   pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                          (GDestroyNotify)g_free, NULL);
+  pull_data->dir = g_strdup (dir_to_pull);
 
   pull_data->start_time = g_get_monotonic_time ();
 
@@ -1209,7 +1304,7 @@ ostree_repo_pull (OstreeRepo               *self,
             {
               if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
                 goto out;
-      
+
               /* Transfer ownership of contents */
               g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
             }
@@ -1241,6 +1336,12 @@ ostree_repo_pull (OstreeRepo               *self,
         }
     }
 
+  /* Create the state directory here - it's new with the commitpartial code,
+   * and may not exist in older repositories.
+   */
+  if (!gs_file_ensure_directory (pull_data->repo->state_dir, FALSE, pull_data->cancellable, error))
+    goto out;
+
   pull_data->phase = OSTREE_PULL_PHASE_FETCHING_OBJECTS;
 
   if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming,
@@ -1328,6 +1429,27 @@ ostree_repo_pull (OstreeRepo               *self,
       ostree_async_progress_set_status (pull_data->progress, msg);
     }
 
+  /* iterate over commits fetched and delete any commitpartial files */
+  if (!dir_to_pull)
+    {
+      g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
+      while (g_hash_table_iter_next (&hash_iter, &key, &value))
+        {
+          const char *checksum = value;
+          gs_unref_object GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, checksum);
+          if (!ot_gfile_ensure_unlinked (commitpartial_path, cancellable, error))
+            goto out;
+        }
+        g_hash_table_iter_init (&hash_iter, commits_to_fetch);
+        while (g_hash_table_iter_next (&hash_iter, &key, &value))
+          {
+            const char *commit = value;
+            gs_unref_object GFile *commitpartial_path = get_commitpartial_path (pull_data->repo, commit);
+            if (!ot_gfile_ensure_unlinked (commitpartial_path, cancellable, error))
+              goto out;
+          }
+    }
+
   ret = TRUE;
  out:
   if (pull_data->main_context)

src/libostree/ostree-repo.c

@@ -100,6 +100,7 @@ ostree_repo_finalize (GObject *object)
   if (self->objects_dir_fd != -1)
     (void) close (self->objects_dir_fd);
   g_clear_object (&self->deltas_dir);
+  g_clear_object (&self->state_dir);
   g_clear_object (&self->uncompressed_objects_dir);
   if (self->uncompressed_objects_dir_fd != -1)
     (void) close (self->uncompressed_objects_dir_fd);
@@ -178,6 +179,7 @@ ostree_repo_constructed (GObject *object)
   self->uncompressed_objects_dir = g_file_resolve_relative_path (self->repodir, "uncompressed-objects-cache/objects");
   self->deltas_dir = g_file_get_child (self->repodir, "deltas");
   self->uncompressed_objects_dir = g_file_get_child (self->repodir, "uncompressed-objects-cache");
+  self->state_dir = g_file_get_child (self->repodir, "state");
   self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache");
   self->config_file = g_file_get_child (self->repodir, "config");
 
@@ -586,6 +588,11 @@ ostree_repo_create (OstreeRepo     *self,
   if (!g_file_make_directory (grandchild, cancellable, error))
     goto out;
 
+  g_clear_object (&child);
+  child = g_file_get_child (self->repodir, "state");
+  if (!g_file_make_directory (child, cancellable, error))
+    goto out;
+
   if (!ostree_repo_open (self, cancellable, error))
     goto out;
 

src/libostree/ostree-repo.h

@@ -538,6 +538,16 @@ gboolean ostree_repo_pull (OstreeRepo             *self,
                            GCancellable           *cancellable,
                            GError                **error);
 
+gboolean
+ostree_repo_pull_one_dir (OstreeRepo               *self,
+                          const char               *remote_name,
+                          const char               *dir_to_pull,
+                          char                    **refs_to_fetch,
+                          OstreeRepoPullFlags       flags,
+                          OstreeAsyncProgress      *progress,
+                          GCancellable             *cancellable,
+                          GError                  **error);
+
 gboolean ostree_repo_sign_commit (OstreeRepo     *self,
                                   const gchar    *commit_checksum,
                                   const gchar    *key_id,

src/libostree/ostree-sysroot-upgrader.c

@@ -427,6 +427,27 @@ ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader  *self,
                               gboolean               *out_changed,
                               GCancellable           *cancellable,
                               GError                **error)
+{
+  return ostree_sysroot_upgrader_pull_one_dir (self, NULL, flags, upgrader_flags, progress, out_changed, cancellable, error);
+}
+
+/**
+ * ostree_sysroot_upgrader_pull_one_dir:
+ *
+ * Like ostree_sysroot_upgrader_pull(), but allows retrieving just a
+ * subpath of the tree.  This can be used to download metadata files
+ * from inside the tree such as package databases.
+ *
+ */
+gboolean
+ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader  *self,
+                                      const char             *dir_to_pull,
+                                      OstreeRepoPullFlags     flags,
+                                      OstreeSysrootUpgraderPullFlags     upgrader_flags,
+                                      OstreeAsyncProgress    *progress,
+                                      gboolean               *out_changed,
+                                      GCancellable           *cancellable,
+                                      GError                **error)
 {
   gboolean ret = FALSE;
   gs_unref_object OstreeRepo *repo = NULL;
@@ -448,7 +469,7 @@ ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader  *self,
 
   if (self->origin_remote)
     {
-      if (!ostree_repo_pull (repo, self->origin_remote, refs_to_fetch,
+      if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch,
                              flags, progress,
                              cancellable, error))
         goto out;

src/libostree/ostree-sysroot-upgrader.h

@@ -65,6 +65,15 @@ gboolean ostree_sysroot_upgrader_pull (OstreeSysrootUpgrader  *self,
                                        GCancellable           *cancellable,
                                        GError                **error);
 
+gboolean ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader  *self,
+                                      const char                   *dir_to_pull,
+                                      OstreeRepoPullFlags     flags,
+                                      OstreeSysrootUpgraderPullFlags     upgrader_flags,
+                                      OstreeAsyncProgress    *progress,
+                                      gboolean               *out_changed,
+                                      GCancellable           *cancellable,
+                                      GError                **error);
+
 gboolean ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader  *self,
                                          GCancellable           *cancellable,
                                          GError                **error);

src/ostree/ot-builtin-pull.c

@@ -29,12 +29,14 @@
 
 static gboolean opt_disable_fsync;
 static gboolean opt_mirror;
-
+static char* opt_subpath;
-static GOptionEntry options[] = {
+ 
-  { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL },
+ static GOptionEntry options[] = {
-  { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
+   { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL },
-  { NULL }
+   { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
-};
+   { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL },
+   { NULL }
+ };
 
 gboolean
 ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error)
@@ -95,8 +97,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *canc
       progress = ostree_async_progress_new_and_connect (ot_common_pull_progress, console);
     }
 
-  if (!ostree_repo_pull (repo, remote, refs_to_fetch ? (char**)refs_to_fetch->pdata : NULL,
+  if (!ostree_repo_pull_one_dir (repo, remote, opt_subpath,
-                         pullflags, progress, cancellable, error))
+                                  refs_to_fetch ? (char**)refs_to_fetch->pdata : NULL,
+                                  pullflags, progress, cancellable, error))
     goto out;
 
   if (progress)

tests/test-pull-subpath.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+
+echo '1..1'
+
+echo "SUBDIR TEST"
+
+repopath=${test_tmpdir}/ostree-srv/gnomerepo
+cp -a ${repopath} ${repopath}.orig
+
+cd ${test_tmpdir}
+rm repo -rf
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+${CMD_PREFIX} ostree --repo=repo pull --subpath=/baz origin main
+
+${CMD_PREFIX} ostree --repo=repo ls origin:main /baz
+if ${CMD_PREFIX} ostree --repo=repo ls origin:main /firstfile 2>err.txt; then
+    assert_not_reached
+fi
+assert_file_has_content err.txt "Couldn't find file object"
+rev=$(ostree --repo=repo rev-parse origin:main)
+assert_has_file repo/state/${rev}.commitpartial
+
+${CMD_PREFIX} ostree --repo=repo pull origin main
+assert_not_has_file repo/state/${rev}.commitpartial
+${CMD_PREFIX} ostree --repo=repo fsck