pull: Add depth support

For mirroring in particular, we really want to be able to traverse
all history.

$ ostree --repo=repo pull --mirror --depth=-1

https://bugzilla.gnome.org/show_bug.cgi?id=739240

Makefile-tests.am

@@ -31,6 +31,7 @@ testfiles = test-basic \
 	test-libarchive \
 	test-pull-archive-z \
 	test-pull-corruption \
+	test-pull-depth \
 	test-pull-mirror-summary \
 	test-pull-large-metadata \
 	test-pull-metalink \

src/libostree/ostree-repo-pull.c

@@ -55,6 +55,7 @@ typedef struct {
   GVariant         *summary;
   GPtrArray        *static_delta_metas;
   GHashTable       *expected_commit_sizes; /* Maps commit checksum to known size */
+  GHashTable       *commit_to_depth; /* Maps commit checksum maximum depth */
   GHashTable       *scanned_metadata; /* Maps object name to itself */
   GHashTable       *requested_metadata; /* Maps object name to itself */
   GHashTable       *requested_content; /* Maps object name to itself */
@@ -67,6 +68,7 @@ typedef struct {
   guint             n_fetched_metadata;
   guint             n_fetched_content;
 
+  int               maxdepth;
   guint64           start_time;
 
   char         *dir;
@@ -855,9 +857,13 @@ scan_commit_object (OtPullData         *pull_data,
                     GError            **error)
 {
   gboolean ret = FALSE;
+  gboolean have_parent;
   gs_unref_variant GVariant *commit = NULL;
+  gs_unref_variant GVariant *parent_csum = NULL;
   gs_unref_variant GVariant *tree_contents_csum = NULL;
   gs_unref_variant GVariant *tree_meta_csum = NULL;
+  gpointer depthp;
+  gint depth;
 
   if (recursion_depth > OSTREE_MAX_RECURSION)
     {
@@ -866,6 +872,18 @@ scan_commit_object (OtPullData         *pull_data,
       goto out;
     }
 
+  if (g_hash_table_lookup_extended (pull_data->commit_to_depth, checksum,
+                                    NULL, &depthp))
+    {
+      depth = GPOINTER_TO_INT (depthp);
+    }
+  else
+    {
+      depth = pull_data->maxdepth;
+      g_hash_table_insert (pull_data->commit_to_depth, g_strdup (checksum),
+                           GINT_TO_POINTER (depth));
+    }
+
 #ifdef HAVE_GPGME
   if (pull_data->gpg_verify)
     {
@@ -884,6 +902,46 @@ scan_commit_object (OtPullData         *pull_data,
     goto out;
 
   /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
+  g_variant_get_child (commit, 1, "@ay", &parent_csum);
+  have_parent = g_variant_n_children (parent_csum) > 0;
+  if (have_parent && pull_data->maxdepth == -1)
+    {
+      if (!scan_one_metadata_object_c (pull_data,
+                                       ostree_checksum_bytes_peek (parent_csum),
+                                       OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
+                                       cancellable, error))
+        goto out;
+    }
+  else if (have_parent && depth > 0)
+    {
+      char parent_checksum[65];
+      gpointer parent_depthp;
+      int parent_depth;
+
+      ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (parent_csum), parent_checksum);
+  
+      if (g_hash_table_lookup_extended (pull_data->commit_to_depth, parent_checksum,
+                                        NULL, &parent_depthp))
+        {
+          parent_depth = GPOINTER_TO_INT (parent_depthp);
+        }
+      else
+        {
+          parent_depth = depth - 1;
+        }
+
+      if (parent_depth >= 0)
+        {
+          g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum),
+                               GINT_TO_POINTER (parent_depth));
+          if (!scan_one_metadata_object_c (pull_data,
+                                           ostree_checksum_bytes_peek (parent_csum),
+                                           OSTREE_OBJECT_TYPE_COMMIT, recursion_depth + 1,
+                                           cancellable, error))
+            goto out;
+        }
+    }
+
   g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
   g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
 
@@ -969,6 +1027,15 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
               do_scan = TRUE;
               pull_data->commitpartial_exists = TRUE;
             }
+          else if (pull_data->maxdepth != 0)
+            {
+              /* Not fully accurate, but the cost here of scanning all
+               * input commit objects if we're doing a depth fetch is
+               * pretty low.  We'll do more accurate handling of depth
+               * when parsing the actual commit.
+               */
+              do_scan = TRUE;
+            }
         }
 
       if (do_scan)
@@ -1268,8 +1335,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       /* Reduce risk of issues if enum happens to be 64 bit for some reason */
       flags = flags_i;
       (void) g_variant_lookup (options, "subdir", "&s", &dir_to_pull);
+      (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth);
     }
 
+  g_return_val_if_fail (pull_data->maxdepth < -1, FALSE);
+
   if (dir_to_pull)
     g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
 
@@ -1286,6 +1356,9 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   pull_data->expected_commit_sizes = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                             (GDestroyNotify)g_free,
                                                             (GDestroyNotify)g_free);
+  pull_data->commit_to_depth = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                                      (GDestroyNotify)g_free,
+                                                      NULL);
   pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
                                                        (GDestroyNotify)g_variant_unref, NULL);
   pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
@@ -1697,6 +1770,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     soup_uri_free (pull_data->base_uri);
   g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
   g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref);
+  g_clear_pointer (&pull_data->commit_to_depth, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);

src/libostree/ostree-repo.c

@@ -2207,6 +2207,7 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
  *   * subdir (s): Pull just this subdirectory
  *   * flags (i): An instance of #OstreeRepoPullFlags
  *   * refs: (as): Array of string refs
+ *   * depth: (i): How far in the history to traverse; default is 0, -1 means infinite
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,

src/ostree/ot-builtin-pull.c

@@ -30,11 +30,13 @@
 static gboolean opt_disable_fsync;
 static gboolean opt_mirror;
 static char* opt_subpath;
+static int opt_depth = 0;
  
  static GOptionEntry options[] = {
    { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL },
    { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
    { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL },
+   { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" },
    { NULL }
  };
 
@@ -97,10 +99,25 @@ ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *canc
       progress = ostree_async_progress_new_and_connect (ot_common_pull_progress, console);
     }
 
-  if (!ostree_repo_pull_one_dir (repo, remote, opt_subpath,
-                                  refs_to_fetch ? (char**)refs_to_fetch->pdata : NULL,
-                                  pullflags, progress, cancellable, error))
-    goto out;
+  {
+    GVariantBuilder builder;
+    g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}"));
+
+    if (opt_subpath)
+      g_variant_builder_add (&builder, "{s@v}", "subdir",
+                             g_variant_new_variant (g_variant_new_string (opt_subpath)));
+    g_variant_builder_add (&builder, "{s@v}", "flags",
+                           g_variant_new_variant (g_variant_new_int32 (pullflags)));
+    if (refs_to_fetch)
+      g_variant_builder_add (&builder, "{s@v}", "refs",
+                             g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch->pdata, -1)));
+    g_variant_builder_add (&builder, "{s@v}", "depth",
+                           g_variant_new_variant (g_variant_new_int32 (opt_depth)));
+    
+    if (!ostree_repo_pull_with_options (repo, remote, g_variant_builder_end (&builder),
+                                        progress, cancellable, error))
+      goto out;
+  }
 
   if (progress)
     ostree_async_progress_finish (progress);

tests/test-pull-depth.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+
+echo '1..1'
+
+cd ${test_tmpdir}
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+ostree --repo=repo pull --depth=0 origin main
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^1$"
+
+ostree --repo=repo pull --depth=0 origin main
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^1$"
+
+ostree --repo=repo pull --depth=1 origin main
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^2$"
+
+ostree --repo=repo pull --depth=1 origin main
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^2$"
+
+ostree --repo=repo pull --depth=-1 origin main
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^3$"
+
+echo "ok pull depth"