Add an experimental option to use libostree's "staging" API

Now that infrastructure for this has landed in libostree,
let's make it easy for people to opt-in to testing it.  This is a distinct first
step for adding it as an update policy.

Closes: #1352
Approved by: jlebon

src/app/rpmostree-builtin-status.c

@@ -604,6 +604,12 @@ print_one_deployment (RPMOSTreeSysroot *sysroot_proxy,
         g_print ("%s%s", get_bold_end (), get_red_end ());
     }
 
+  gboolean is_staged = FALSE;
+  g_variant_dict_lookup (dict, "staged", "b", &is_staged);
+
+  if (opt_verbose && (is_staged || first))
+    rpmostree_print_kv ("Staged", max_key_len, is_staged ? "yes" : "no");
+
   /* This used to be OSName; see https://github.com/ostreedev/ostree/pull/794 */
   if (opt_verbose || have_multiple_stateroots)
     rpmostree_print_kv ("StateRoot", max_key_len, os_name);

src/daemon/rpmostree-sysroot-upgrader.c

@@ -29,6 +29,8 @@
 #include "rpmostree-core.h"
 #include "rpmostree-origin.h"
 #include "rpmostree-kernel.h"
+#include "rpmostreed-daemon.h"
+#include "rpmostree-kernel.h"
 #include "rpmostree-rpm-util.h"
 #include "rpmostree-postprocess.h"
 #include "rpmostree-output.h"
@@ -1195,19 +1197,42 @@ rpmostree_sysroot_upgrader_deploy (RpmOstreeSysrootUpgrader *self,
   const char *target_revision = self->final_revision ?: self->base_revision;
   g_assert (target_revision);
 
+  /* Use staging only if we're booted into the target root.  Further,
+   * it's currently gated behind an experimental flag.
+   */
+  const gboolean use_staging =
+    ostree_sysroot_get_booted_deployment (self->sysroot) &&
+    rpmostreed_get_ex_stage_deployments (rpmostreed_daemon_get ());
+
   g_autoptr(GKeyFile) origin = rpmostree_origin_dup_keyfile (self->origin);
   g_autoptr(OstreeDeployment) new_deployment = NULL;
-  if (!ostree_sysroot_deploy_tree (self->sysroot, self->osname,
+  if (use_staging)
-                                   target_revision, origin,
+    {
-                                   self->cfg_merge_deployment,
+      if (!ostree_sysroot_stage_tree (self->sysroot, self->osname,
-                                   self->kargs_strv,
+                                      target_revision, origin,
-                                   &new_deployment,
+                                      self->cfg_merge_deployment,
-                                   cancellable, error))
+                                      self->kargs_strv,
-    return FALSE;
+                                      &new_deployment,
+                                      cancellable, error))
+        return FALSE;
+    }
+  else
+    {
+      if (!ostree_sysroot_deploy_tree (self->sysroot, self->osname,
+                                       target_revision, origin,
+                                       self->cfg_merge_deployment,
+                                       self->kargs_strv,
+                                       &new_deployment,
+                                       cancellable, error))
+        return FALSE;
+    }
 
   /* Also do a sanitycheck even if there's no local mutation; it's basically free
    * and might save someone in the future.  The RPMOSTREE_SKIP_SANITYCHECK
    * environment variable is just used by test-basic.sh currently.
+   *
+   * Note that since the staging changes, this now operates without a
+   * config-merged state.
    */
   if (!self->final_revision)
     {
@@ -1221,10 +1246,9 @@ rpmostree_sysroot_upgrader_deploy (RpmOstreeSysrootUpgrader *self,
       if (!rpmostree_deployment_sanitycheck (deployment_dfd, cancellable, error))
         return FALSE;
     }
-
+  else
-  if (self->final_revision)
     {
-      /* Generate a temporary ref for the new deployment in case we are
+      /* Generate a temporary ref for the base revision in case we are
        * interrupted; the base layer refs generation isn't transactional.
        */
       if (!ostree_repo_set_ref_immediate (self->repo, NULL, RPMOSTREE_TMP_BASE_REF,
@@ -1233,10 +1257,22 @@ rpmostree_sysroot_upgrader_deploy (RpmOstreeSysrootUpgrader *self,
         return FALSE;
     }
 
-  if (!rpmostree_syscore_write_deployment (self->sysroot, new_deployment,
+  if (use_staging)
-                                           self->cfg_merge_deployment, FALSE,
+    {
-                                           cancellable, error))
+      /* In the staging path, we just need to regenerate our baselayer refs and
-    return FALSE;
+       * do the prune.  The stage_tree() API above should have loaded our new deployment
+       * into the set.
+       */
+      if (!rpmostree_syscore_cleanup (self->sysroot, self->repo, cancellable, error))
+        return FALSE;
+    }
+  else
+    {
+      if (!rpmostree_syscore_write_deployment (self->sysroot, new_deployment,
+                                               self->cfg_merge_deployment, FALSE,
+                                               cancellable, error))
+        return FALSE;
+    }
 
   if (out_deployment)
     *out_deployment = g_steal_pointer (&new_deployment);

src/daemon/rpmostreed-daemon.c

@@ -33,6 +33,7 @@
 
 #define RPMOSTREED_CONF SYSCONFDIR "/rpm-ostreed.conf"
 #define DAEMON_CONFIG_GROUP "Daemon"
+#define EXPERIMENTAL_CONFIG_GROUP "Experimental"
 
 /**
  * SECTION: daemon
@@ -62,9 +63,10 @@ struct _RpmostreedDaemon {
   RpmostreedSysroot *sysroot;
   gchar *sysroot_path;
 
-  /* we only have two settings for now, so let's just keep it in the main struct */
+  /* Settings from the config file */
   guint idle_exit_timeout;
   RpmostreedAutomaticUpdatePolicy auto_update_policy;
+  gboolean ex_stage_deployments;
 
   GDBusConnection *connection;
   GDBusObjectManagerServer *object_manager;
@@ -349,6 +351,12 @@ rpmostreed_get_automatic_update_policy (RpmostreedDaemon *self)
   return self->auto_update_policy;
 }
 
+gboolean
+rpmostreed_get_ex_stage_deployments (RpmostreedDaemon *self)
+{
+  return self->ex_stage_deployments;
+}
+
 /* in-place version of g_ascii_strdown */
 static inline void
 ascii_strdown_inplace (char *str)
@@ -384,6 +392,9 @@ rpmostreed_daemon_reload_config (RpmostreedDaemon *self,
         return FALSE;
     }
 
+  self->ex_stage_deployments = g_key_file_get_boolean (config, EXPERIMENTAL_CONFIG_GROUP,
+                                                       "StageDeployments", NULL);
+
   /* don't update changed for this; it's contained to RpmostreedDaemon so no other objects
    * need to be reloaded if it changes */
   self->idle_exit_timeout = idle_exit_timeout;

src/daemon/rpmostreed-daemon.h

@@ -54,3 +54,6 @@ gboolean           rpmostreed_daemon_reload_config  (RpmostreedDaemon *self,
 
 RpmostreedAutomaticUpdatePolicy
 rpmostreed_get_automatic_update_policy (RpmostreedDaemon *self);
+
+gboolean
+rpmostreed_get_ex_stage_deployments (RpmostreedDaemon *self);

src/daemon/rpmostreed-deployment-utils.c

@@ -346,6 +346,9 @@ rpmostreed_deployment_generate_variant (OstreeSysroot *sysroot,
   if (live_replaced)
     g_variant_dict_insert (&dict, "live-replaced", "s", live_replaced);
 
+  if (ostree_deployment_is_staged (deployment))
+    g_variant_dict_insert (&dict, "staged", "b", TRUE);
+
   if (refspec)
     g_variant_dict_insert (&dict, "origin", "s", refspec);
 

src/daemon/rpmostreed-os.c

@@ -793,7 +793,7 @@ os_merge_or_start_deployment_txn (RPMOSTreeOS            *interface,
           rpmostreed_transaction_monitor_add (self->transaction_monitor, transaction);
 
           /* For the AutomaticUpdateTrigger "check" case, we want to make sure we refresh
-           * the CachedUpdate property; "deploy" will do this through sysroot_changed */
+           * the CachedUpdate property; "stage" will do this through sysroot_changed */
           const char *method_name = g_dbus_method_invocation_get_method_name (invocation);
           if (g_str_equal (method_name, "AutomaticUpdateTrigger") &&
               (default_flags & (RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_ONLY |
@@ -1794,6 +1794,7 @@ rpmostreed_os_load_internals (RpmostreedOS *self, GError **error)
   OstreeSysroot *ot_sysroot = rpmostreed_sysroot_get_root (rpmostreed_sysroot_get ());
   OstreeRepo *ot_repo = rpmostreed_sysroot_get_repo (rpmostreed_sysroot_get ());
 
+  /* Booted */
   g_autofree gchar* booted_id = NULL;
   OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (ot_sysroot);
   g_autoptr(GVariant) booted_variant = NULL; /* Strong ref as we reuse it below */
@@ -1810,6 +1811,7 @@ rpmostreed_os_load_internals (RpmostreedOS *self, GError **error)
     booted_variant = g_variant_ref_sink (rpmostreed_deployment_generate_blank_variant ());
   rpmostree_os_set_booted_deployment (RPMOSTREE_OS (self), booted_variant);
 
+  /* Default (pending or booted) and rollback */
   g_autoptr(OstreeDeployment) rollback_deployment = NULL;
   g_autoptr(OstreeDeployment) pending_deployment = NULL;
   ostree_sysroot_query_deployments_for (ot_sysroot, name,

src/daemon/rpmostreed-transaction-types.c

@@ -1107,7 +1107,6 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
         return glnx_throw (error, "Refusing to download rpm-md for offline OS '%s'",
                            self->osname);
 
-
       /* XXX: in rojig mode we'll want to do this unconditionally */
       g_autoptr(DnfSack) sack = NULL;
       if (g_hash_table_size (rpmostree_origin_get_packages (origin)) > 0)

tests/vmcheck/test-autoupdate.sh

@@ -40,6 +40,9 @@ vm_rpmostree rebase vmcheckmote:vmcheck \
   --install layered-sec-none \
   --install layered-sec-low \
   --install layered-sec-crit
+if vm_cmd 'grep -qE -e "^StageDeployments=true" /etc/rpm-ostreed.conf'; then
+    vm_cmd systemctl is-active ostree-finalize-staged.service
+fi
 vm_reboot
 vm_rpmostree status -v
 vm_assert_status_jq \

tests/vmcheck/test-meta-staged.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright (C) 2018 Red Hat
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+dn=$(cd $(dirname $0) && pwd)
+
+. ${commondir}/libtest.sh
+. ${commondir}/libvm.sh
+
+set -x
+
+# This test suite enables the libostree staging feature, and executes
+# the upgrade and layering-relayer tests.
+
+export VMCHECK_FLAGS=stage-deployments
+vm_cmd 'echo "[Experimental]" >> /etc/rpm-ostreed.conf'
+vm_cmd 'echo StageDeployments=true >> /etc/rpm-ostreed.conf'
+vm_rpmostree reload
+
+#${dn}/test-upgrades.sh
+
+${dn}/test-layering-relayer.sh

tests/vmcheck/test.sh

@@ -76,6 +76,11 @@ if vm_cmd test -f /etc/rpm-ostreed.conf; then
 fi
 if vm_cmd test -f /usr/etc/rpm-ostreed.conf; then
     vm_cmd cp -f /usr/etc/rpm-ostreed.conf /etc
+    # Unless we're doing overrides
+    if [[ "${VMCHECK_FLAGS:-}" =~ "stage-deployments" ]]; then
+        vm_cmd 'echo "[Experimental]" >> /etc/rpm-ostreed.conf'
+        vm_cmd 'echo StageDeployments=true >> /etc/rpm-ostreed.conf'
+    fi
 fi
 
 vm_cmd ostree remote delete --if-exists vmcheckmote