Add "pull --localcache-repo"

This is a lot like `git clone --reference`, but we chose "localcache" as the
term "reference" is already used.

The main use case I'm targeting this for is the Fedora Atomic Host installer
case where we embed the repo content in the installer, but we may want to
kickstart and download newer content. There, while we want to get a newer ref,
we can still use the local repo as an object cache, since we have it sitting
there in memory anyways.

Another case is where one has a host ostree (say e.g. Fedora Atomic
Workstation), and one wants to create a local archive mirror of FAH. Then one
can use `pull --reference /ostree/repo` and pull the common objects (e.g.
contents of `bash.rpm` etc.)

Closes: https://github.com/ostreedev/ostree/issues/975

Closes: #982
Approved by: jlebon

Makefile-tests.am

@@ -78,6 +78,7 @@ _installed_or_uninstalled_test_scripts = \
 	tests/test-pull-repeated.sh \
 	tests/test-pull-untrusted.sh \
 	tests/test-pull-override-url.sh \
+	tests/test-pull-localcache.sh \
 	tests/test-local-pull.sh \
 	tests/test-local-pull-depth.sh \
 	tests/test-gpg-signed-commit.sh \

man/ostree-pull.xml

@@ -73,6 +73,16 @@ Boston, MA 02111-1307, USA.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+              <term><option>--localcache-repo</option></term>
+
+              <listitem><para>
+                Like git's <literal>clone --reference</literal>.  Reuse the provided
+                OSTree repo as a local object cache of objects when doing HTTP fetches.
+                May be specified multiple times.
+              </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--untrusted</option></term>
 

src/libostree/ostree-repo-pull.c

@@ -64,6 +64,7 @@ typedef struct {
   GPtrArray     *meta_mirrorlist;    /* List of base URIs for fetching metadata */
   GPtrArray     *content_mirrorlist; /* List of base URIs for fetching content */
   OstreeRepo   *remote_repo_local;
+  GPtrArray    *localcache_repos; /* Array<OstreeRepo> */
 
   GMainContext    *main_context;
   GCancellable *cancellable;
@@ -118,6 +119,9 @@ typedef struct {
   guint             n_fetched_deltapart_fallbacks;
   guint             n_fetched_metadata;
   guint             n_fetched_content;
+  /* Objects from pull --localcache-repo */
+  guint             n_fetched_localcache_metadata;
+  guint             n_fetched_localcache_content;
 
   int               maxdepth;
   guint64           start_time;
@@ -236,6 +240,8 @@ update_progress (gpointer user_data)
                              "scanned-metadata", "u", n_scanned_metadata,
                              "bytes-transferred", "t", bytes_transferred,
                              "start-time", "t", start_time,
+                             "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata,
+                             "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content,
                              /* Deltas */
                              "fetched-delta-parts",
                                   "u", pull_data->n_fetched_deltaparts,
@@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data,
  */
 static gboolean
 import_one_local_content_object (OtPullData *pull_data,
+                                 OstreeRepo *src_repo,
                                  const char *checksum,
                                  GCancellable *cancellable,
                                  GError    **error)
 {
-  g_assert (pull_data->remote_repo_local);
-
   const gboolean trusted = !pull_data->is_untrusted;
   if (trusted && !pull_data->is_bareuseronly_files)
     {
-      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local,
+      if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo,
                                                       OSTREE_OBJECT_TYPE_FILE, checksum,
                                                       trusted,
                                                       cancellable, error))
@@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data,
       g_autoptr(GFileInfo) content_finfo = NULL;
       g_autoptr(GVariant) content_xattrs = NULL;
 
-      if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum,
+      if (!ostree_repo_load_file (src_repo, checksum,
                                   &content_input, &content_finfo, &content_xattrs,
                                   cancellable, error))
         return FALSE;
@@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData   *pull_data,
       /* Is this a local repo? */
       if (pull_data->remote_repo_local)
         {
-          if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error))
+          if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local,
+                                                file_checksum, cancellable, error))
             return FALSE;
+
+          /* Note early loop continue */
+          continue;
         }
-      else
+
+      /* We're doing HTTP, but see if we have the object in a local cache first */
+      gboolean did_import_from_cache_repo = FALSE;
+      if (pull_data->localcache_repos)
         {
-          /* In this case we're doing HTTP pulls */
-          g_hash_table_add (pull_data->requested_content, file_checksum);
-          enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
-          file_checksum = NULL;  /* Transfer ownership */
+          for (guint j = 0; j < pull_data->localcache_repos->len; j++)
+            {
+              OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j];
+              gboolean localcache_repo_has_obj;
+
+              if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
+                                           &localcache_repo_has_obj, cancellable, error))
+                return FALSE;
+              if (!localcache_repo_has_obj)
+                continue;
+              if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum,
+                                                    cancellable, error))
+                return FALSE;
+              did_import_from_cache_repo = TRUE;
+              pull_data->n_fetched_localcache_content++;
+              break;
+            }
         }
+      if (did_import_from_cache_repo)
+        continue; /* Note early continue */
+
+      /* Not available locally, queue a HTTP request */
+      g_hash_table_add (pull_data->requested_content, file_checksum);
+      enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE);
+      file_checksum = NULL;  /* Transfer ownership */
     }
 
   g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1);
@@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
       is_stored = TRUE;
       is_requested = TRUE;
     }
+  /* Do we have any localcache repos? */
+  else if (!is_stored && pull_data->localcache_repos)
+    {
+      for (guint i = 0; i < pull_data->localcache_repos->len; i++)
+        {
+          OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i];
+          gboolean localcache_repo_has_obj;
+
+          if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum,
+                                       &localcache_repo_has_obj, cancellable, error))
+            return FALSE;
+          if (!localcache_repo_has_obj)
+            continue;
+          if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo,
+                                                          objtype, tmp_checksum,
+                                                          !pull_data->is_untrusted,
+                                                          cancellable, error))
+            return FALSE;
+          /* See comment above */
+          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+            g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum));
+          is_stored = TRUE;
+          is_requested = TRUE;
+          pull_data->n_fetched_localcache_metadata++;
+          break;
+        }
+    }
 
   if (!is_stored && !is_requested)
     {
@@ -2865,6 +2924,7 @@ initiate_request (OtPullData                 *pull_data,
  *   * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction.
  *   * http-headers (a(ss)): Additional headers to add to all HTTP requests
  *   * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
+ *   * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches
  */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   gboolean inherit_transaction = FALSE;
   g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL;  /* (element-type OstreeCollectionRef utf8) */
   int i;
+  g_autofree char **opt_localcache_repos = NULL;
 
   if (options)
     {
@@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
       (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
       (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
+      (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                              (GDestroyNotify)fetch_object_data_free);
   pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
 
+  if (opt_localcache_repos && *opt_localcache_repos)
+    {
+      pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref);
+      for (char **it = opt_localcache_repos; it && *it; it++)
+        {
+          const char *localcache_path = *it;
+          g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path);
+          g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file);
+          if (!ostree_repo_open (cacherepo, cancellable, error))
+            goto out;
+          g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo));
+        }
+    }
+
   if (dir_to_pull != NULL || dirs_to_pull != NULL)
     {
       pull_data->dirs = g_ptr_array_new_with_free_func (g_free);
@@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       else
         g_string_append_printf (buf, "%u metadata, %u content objects fetched",
                                 pull_data->n_fetched_metadata, pull_data->n_fetched_content);
+      if (pull_data->n_fetched_localcache_metadata ||
+          pull_data->n_fetched_localcache_content)
+        g_string_append_printf (buf, " (%u meta, %u content local)",
+                                pull_data->n_fetched_localcache_metadata,
+                                pull_data->n_fetched_localcache_content);
 
       g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds",
                               (guint64)(bytes_transferred / shift),
@@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_object (&pull_data->fetcher);
   g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref);
   g_clear_object (&pull_data->cancellable);
+  g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);
   g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);

src/ostree/ot-builtin-pull.c

@@ -41,6 +41,7 @@ static char* opt_cache_dir;
 static int opt_depth = 0;
 static int opt_frequency = 0;
 static char* opt_url;
+static char** opt_localcache_repos;
 
 static GOptionEntry options[] = {
    { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL },
@@ -57,6 +58,7 @@ static GOptionEntry options[] = {
    { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL },
    { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" },
    { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" },
+   { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" },
    { NULL }
  };
 
@@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
     if (override_commit_ids)
       g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
+    if (opt_localcache_repos)
+      g_variant_builder_add (&builder, "{s@v}", "localcache-repos",
+                             g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1)));
 
     if (opt_http_headers)
       {

tests/test-pull-localcache.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive"
+
+echo '1..1'
+
+cd ${test_tmpdir}
+gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo"
+ostree_repo_init repo --mode "archive"
+ostree_repo_init repo-local --mode "archive"
+for repo in repo{,-local}; do
+    ${CMD_PREFIX} ostree --repo=${repo} remote add --set=gpg-verify=false origin ${gnomerepo_url}
+done
+
+# Pull the contents to our local cache
+${CMD_PREFIX} ostree --repo=repo-local pull origin main
+rm files -rf
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main files
+echo anewfile > files/anewfile
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=dir=files
+
+${CMD_PREFIX} ostree --repo=repo pull -L repo-local origin main >out.txt
+assert_file_has_content out.txt '3 metadata, 1 content objects fetched (4 meta, 5 content local)'
+echo "ok pull --reference"