pull: Add a --dry-run option for static deltas

One of the design goals with deltas was not just wire efficiency, but also having all the data up front about how much data would be transferred before starting. Let's expose that better by adding a `dry-run` option to the pull API. This requires static deltas to be useful. Basically we simply call the progress callback once with the data from the superblock.
2025-01-09 01:18:35 +03:00 · 2016-02-19 12:28:07 -05:00 · 2016-02-19 12:28:07 -05:00 · 4beb5f4eaf
commit 4beb5f4eaf
parent f2c5ecb996
3 changed files with 113 additions and 7 deletions
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@ -48,6 +48,8 @@ typedef struct {
  GCancellable *cancellable;
  OstreeAsyncProgress *progress;

+  gboolean      dry_run;
+  gboolean      dry_run_emitted_progress;
  gboolean      legacy_transaction_resuming;
  enum {
    OSTREE_PULL_PHASE_FETCHING_REFS,
@ -78,6 +80,7 @@ typedef struct {
  guint             n_outstanding_deltapart_write_requests;
  guint             n_total_deltaparts;
  guint64           total_deltapart_size;
+  guint64           total_deltapart_usize;
  gint              n_requested_metadata;
  gint              n_requested_content;
  guint             n_fetched_deltaparts;
@ -227,6 +230,8 @@ update_progress (gpointer user_data)
                                  pull_data->n_total_deltaparts);
  ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size",
                                    pull_data->total_deltapart_size);
+  ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize",
+                                    pull_data->total_deltapart_usize);
  ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks",
                                  pull_data->static_delta_superblocks->len);

@ -243,6 +248,9 @@ update_progress (gpointer user_data)
  else
    ostree_async_progress_set_status (pull_data->progress, NULL);

+  if (pull_data->dry_run)
+    pull_data->dry_run_emitted_progress = TRUE;
+
  return TRUE;
 }

@ -262,6 +270,9 @@ pull_termination_condition (OtPullData          *pull_data)
  if (pull_data->caught_error)
    return TRUE;

+  if (pull_data->dry_run)
+    return pull_data->dry_run_emitted_progress;
+
  switch (pull_data->phase)
    {
    case OSTREE_PULL_PHASE_FETCHING_REFS:
@ -1451,11 +1462,18 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
  if (!ostree_validate_structureof_csum_v (csum_v, error))
    goto out;

+  pull_data->total_deltapart_size += compressed_size;
+  pull_data->total_deltapart_usize += uncompressed_size;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
+
  objtype = (OstreeObjectType)objtype_y;
  checksum = ostree_checksum_from_bytes_v (csum_v);

-  pull_data->total_deltapart_size += compressed_size;
-
  if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
                               &is_stored,
                               cancellable, error))
@ -1524,6 +1542,7 @@ process_one_static_delta (OtPullData   *pull_data,
    }

  /* Write the to-commit object */
+  if (!pull_data->dry_run)
  {
    g_autoptr(GVariant) to_csum_v = NULL;
    g_autofree char *to_checksum = NULL;
@ -1626,7 +1645,11 @@ process_one_static_delta (OtPullData   *pull_data,
      }

      pull_data->total_deltapart_size += size;
+      pull_data->total_deltapart_usize += usize;

+      if (pull_data->dry_run)
+        continue;
+      
      fetch_data = g_new0 (FetchStaticDeltaData, 1);
      fetch_data->pull_data = pull_data;
      fetch_data->objects = g_variant_ref (objects);
@ -1780,6 +1803,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
      (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
      (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
      (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
+      (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
    }

  g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@ -1790,6 +1814,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
    g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);

  g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
+  /* We only do dry runs with static deltas, because we don't really have any
+   * in-advance information for bare fetches.
+   */
+  g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);

  pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
  pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@ -2243,7 +2271,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,

  if (pull_data->progress)
    {
-      update_timeout = g_timeout_source_new_seconds (1);
+      update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
      g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
      g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
      g_source_attach (update_timeout, pull_data->main_context);
@ -2256,6 +2284,12 @@ ostree_repo_pull_with_options (OstreeRepo             *self,

  if (pull_data->caught_error)
    goto out;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
  
  g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0);
  g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0);
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@ -30,6 +30,7 @@
 static gboolean opt_disable_fsync;
 static gboolean opt_mirror;
 static gboolean opt_commit_only;
+static gboolean opt_dry_run;
 static gboolean opt_disable_static_deltas;
 static gboolean opt_require_static_deltas;
 static char* opt_subpath;
@ -42,6 +43,7 @@ static GOptionEntry options[] = {
   { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL },
   { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
   { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL },
+   { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL },
   { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" },
   { NULL }
 };
@ -62,6 +64,39 @@ gpg_verify_result_cb (OstreeRepo *repo,
  gs_console_begin_status_line (console, "", NULL, NULL);
 }

+static gboolean printed_console_progress;
+
+static void
+dry_run_console_progress_changed (OstreeAsyncProgress *progress,
+                                  gpointer             user_data)
+{
+  guint fetched_delta_parts, total_delta_parts;
+  guint64 total_delta_part_size, total_delta_part_usize;
+  GString *buf;
+
+  g_assert (!printed_console_progress);
+  printed_console_progress = TRUE;
+
+  fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts");
+  total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts");
+  total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size");
+  total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize");
+
+  buf = g_string_new ("");
+
+  { g_autofree char *formatted_size =
+      g_format_size (total_delta_part_size);
+    g_autofree char *formatted_usize =
+      g_format_size (total_delta_part_usize);
+
+    g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed",
+                            fetched_delta_parts, total_delta_parts,
+                            formatted_size, formatted_usize);
+  }
+  g_print ("%s\n", buf->str);
+  g_string_free (buf, TRUE);
+}
+
 gboolean
 ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error)
 {
@ -99,6 +134,13 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
  if (opt_commit_only)
    pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY;

+  if (opt_dry_run && !opt_require_static_deltas)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "--dry-run requires --require-static-deltas");
+      goto out;
+    }
+
  if (strchr (argv[1], ':') == NULL)
    {
      remote = g_strdup (argv[1]);
@ -149,11 +191,21 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
      g_ptr_array_add (refs_to_fetch, NULL);
    }

-  console = gs_console_get ();
-  if (console)
+  if (!opt_dry_run)
    {
-      gs_console_begin_status_line (console, "", NULL, NULL);
-      progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console);
+      console = gs_console_get ();
+      if (console)
+        {
+          gs_console_begin_status_line (console, "", NULL, NULL);
+          progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console);
+          signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
+                                                G_CALLBACK (gpg_verify_result_cb),
+                                                console);
+        }
+    }
+  else
+    {
+      progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console);
      signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
                                            G_CALLBACK (gpg_verify_result_cb),
                                            console);
@ -180,6 +232,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
    g_variant_builder_add (&builder, "{s@v}", "require-static-deltas",
                           g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas)));

+    g_variant_builder_add (&builder, "{s@v}", "dry-run",
+                           g_variant_new_variant (g_variant_new_boolean (opt_dry_run)));
+
    if (override_commit_ids)
      g_variant_builder_add (&builder, "{s@v}", "override-commit-ids",
                             g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len)));
@ -192,6 +247,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
  if (progress)
    ostree_async_progress_finish (progress);

+  if (opt_dry_run)
+    g_assert (printed_console_progress);
+
  ret = TRUE;
 out:
  if (signal_handler_id > 0)
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@ -107,12 +107,24 @@ rm main-files -rf
 # Generate delta that we'll use
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^)
+new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
 ostree --repo=ostree-srv/gnomerepo summary -u

+cd ${test_tmpdir}
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
+assert_file_has_content out.txt 'Delta update: 0/1 parts'
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${prev_rev}" "${rev}"
+${CMD_PREFIX} ostree --repo=repo fsck
+
 cd ${test_tmpdir}
 repo_init
 ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
 ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${new_rev}" "${rev}"
 ${CMD_PREFIX} ostree --repo=repo fsck

 cd ${test_tmpdir}
@ -140,6 +152,8 @@ fi
 assert_file_has_content err.txt "deltas required, but none found"
 ${CMD_PREFIX} ostree --repo=repo fsck

+echo "ok delta required but don't exist"
+
 cd ${test_tmpdir}
 rm main-files -rf
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files