Add "ex-stage" update policy, support for ostree staged deployments

Following up to https://github.com/projectatomic/rpm-ostree/pull/1352
AKA 506910d930
which added an experimental flag to globally enable deployment
staging, let's add an `ex-stage` automatic update policy.

I chose to create a new `test-autoupdate-stage.sh` and rename
the previous one to `test-autoupdate-check.sh` in going with
the previous theme of smaller test files; it's
way faster to iterate on new tests when it's a new file. And adding
staging at the top would have been weird.

This was all quite straightforward, just plumbing through lots
of layers.

Closes: #1321
Approved by: jlebon

src/daemon/org.projectatomic.rpmostree1.xml

@@ -56,7 +56,7 @@
     <method name="ReloadConfig">
     </method>
 
-    <!-- none, check -->
+    <!-- none, check, ex-stage -->
     <property name="AutomaticUpdatePolicy" type="s" access="read"/>
 
     <method name="CreateOSName">

src/daemon/rpmostree-sysroot-upgrader.c

@@ -1212,11 +1212,14 @@ rpmostree_sysroot_upgrader_deploy (RpmOstreeSysrootUpgrader *self,
   g_assert (target_revision);
 
   /* Use staging only if we're booted into the target root.  Further,
-   * it's currently gated behind an experimental flag.
+   * it's currently gated behind either the "stage" automatic update
+   * policy, or an experimental flag that applies globally to all operations.
    */
-  const gboolean use_staging =
-    ostree_sysroot_get_booted_deployment (self->sysroot) &&
+  const gboolean staging_is_configured =
+    (self->flags & RPMOSTREE_SYSROOT_UPGRADER_FLAGS_STAGE) > 0 ||
     rpmostreed_get_ex_stage_deployments (rpmostreed_daemon_get ());
+  const gboolean use_staging = staging_is_configured &&
+    ostree_sysroot_get_booted_deployment (self->sysroot) != NULL;
 
   g_autoptr(GKeyFile) origin = rpmostree_origin_dup_keyfile (self->origin);
   g_autoptr(OstreeDeployment) new_deployment = NULL;

src/daemon/rpmostree-sysroot-upgrader.h

@@ -43,6 +43,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (RpmOstreeSysrootUpgrader, g_object_unref)
  * @RPMOSTREE_SYSROOT_UPGRADER_FLAGS_DRY_RUN: Don't deploy new base. If layering packages, only print the transaction
  * @RPMOSTREE_SYSROOT_UPGRADER_FLAGS_PKGCACHE_ONLY: Don't try to update cached packages.
  * @RPMOSTREE_SYSROOT_UPGRADER_FLAGS_SYNTHETIC_PULL: Don't actually pull, just resolve ref and timestamp check
+ * @RPMOSTREE_SYSROOT_UPGRADER_FLAGS_STAGE: Temporary flag to use libostree staging
  *
  * Flags controlling operation of an #RpmOstreeSysrootUpgrader.
  */
@@ -53,6 +54,7 @@ typedef enum {
   RPMOSTREE_SYSROOT_UPGRADER_FLAGS_DRY_RUN              = (1 << 3),
   RPMOSTREE_SYSROOT_UPGRADER_FLAGS_PKGCACHE_ONLY        = (1 << 4),
   RPMOSTREE_SYSROOT_UPGRADER_FLAGS_SYNTHETIC_PULL       = (1 << 5),
+  RPMOSTREE_SYSROOT_UPGRADER_FLAGS_STAGE                = (1 << 6), /* temporary */
 } RpmOstreeSysrootUpgraderFlags;
 
 /* _NONE means we're doing pure ostree, no client-side computation.

src/daemon/rpmostreed-os.c

@@ -793,7 +793,7 @@ os_merge_or_start_deployment_txn (RPMOSTreeOS            *interface,
           rpmostreed_transaction_monitor_add (self->transaction_monitor, transaction);
 
           /* For the AutomaticUpdateTrigger "check" case, we want to make sure we refresh
-           * the CachedUpdate property; "stage" will do this through sysroot_changed */
+           * the CachedUpdate property; "ex-stage" will do this through sysroot_changed */
           const char *method_name = g_dbus_method_invocation_get_method_name (invocation);
           if (g_str_equal (method_name, "AutomaticUpdateTrigger") &&
               (default_flags & (RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_ONLY |
@@ -987,6 +987,9 @@ os_handle_automatic_update_trigger (RPMOSTreeOS *interface,
     case RPMOSTREED_AUTOMATIC_UPDATE_POLICY_CHECK:
       dfault = RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_METADATA_ONLY;
       break;
+    case RPMOSTREED_AUTOMATIC_UPDATE_POLICY_STAGE:
+      dfault = RPMOSTREE_TRANSACTION_DEPLOY_FLAG_STAGE;
+      break;
     default:
       g_assert_not_reached ();
     }

src/daemon/rpmostreed-transaction-types.c

@@ -745,6 +745,8 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
     upgrader_flags |= RPMOSTREE_SYSROOT_UPGRADER_FLAGS_ALLOW_OLDER;
   if (dry_run)
     upgrader_flags |= RPMOSTREE_SYSROOT_UPGRADER_FLAGS_DRY_RUN;
+  if (self->flags & RPMOSTREE_TRANSACTION_DEPLOY_FLAG_STAGE)
+    upgrader_flags |= RPMOSTREE_SYSROOT_UPGRADER_FLAGS_STAGE;
 
   /* DOWNLOAD_METADATA_ONLY isn't directly exposed at the D-Bus API level, so we shouldn't
    * ever run into these conflicting options */

src/daemon/rpmostreed-transaction-types.h

@@ -58,6 +58,7 @@ typedef enum {
   RPMOSTREE_TRANSACTION_DEPLOY_FLAG_CACHE_ONLY = (1 << 7),
   RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_ONLY = (1 << 8),
   RPMOSTREE_TRANSACTION_DEPLOY_FLAG_DOWNLOAD_METADATA_ONLY = (1 << 9),
+  RPMOSTREE_TRANSACTION_DEPLOY_FLAG_STAGE = (1 << 10),
 } RpmOstreeTransactionDeployFlags;
 
 

src/libpriv/rpmostree-types.h

@@ -29,6 +29,7 @@ typedef enum {
 typedef enum {
   RPMOSTREED_AUTOMATIC_UPDATE_POLICY_NONE,
   RPMOSTREED_AUTOMATIC_UPDATE_POLICY_CHECK,
+  RPMOSTREED_AUTOMATIC_UPDATE_POLICY_STAGE,
 } RpmostreedAutomaticUpdatePolicy;
 
 typedef enum {

src/libpriv/rpmostree-util.c

@@ -1045,6 +1045,8 @@ rpmostree_auto_update_policy_to_str (RpmostreedAutomaticUpdatePolicy policy,
       return "none";
     case RPMOSTREED_AUTOMATIC_UPDATE_POLICY_CHECK:
       return "check";
+    case RPMOSTREED_AUTOMATIC_UPDATE_POLICY_STAGE:
+      return "ex-stage";
     default:
       return glnx_null_throw (error, "Invalid policy value %u", policy);
     }
@@ -1060,6 +1062,8 @@ rpmostree_str_to_auto_update_policy (const char *str,
     *out_policy = RPMOSTREED_AUTOMATIC_UPDATE_POLICY_NONE;
   else if (g_str_equal (str, "check"))
     *out_policy = RPMOSTREED_AUTOMATIC_UPDATE_POLICY_CHECK;
+  else if (g_str_equal (str, "ex-stage"))
+    *out_policy = RPMOSTREED_AUTOMATIC_UPDATE_POLICY_STAGE;
   else
     return glnx_throw (error, "Invalid value for AutomaticUpdatePolicy: '%s'", str);
   return TRUE;

tests/common/libvm.sh

@@ -514,6 +514,31 @@ vm_ostreeupdate_prepare() {
 
     vm_ostreeupdate_create v1
     vm_cmd ostree remote add vmcheckmote --no-gpg-verify http://localhost:8888/
+    vm_rpmostree reload
+}
+
+vm_ostreeupdate_prepare_reboot() {
+    vm_ostreeupdate_prepare
+    vm_rpmostree rebase vmcheckmote:vmcheck
+    vm_reboot
+    vm_rpmostree cleanup -pr
+    vm_assert_status_jq ".deployments[0][\"origin\"] == \"vmcheckmote:vmcheck\"" \
+                        ".deployments[0][\"booted\"]" \
+                        ".deployments[0][\"version\"] == \"v1\""
+    vm_rpmostree status > status.txt
+    assert_file_has_content_literal status.txt 'auto updates disabled'
+    # start it up again since we rebooted
+    vm_start_httpd ostree_server $REMOTE_OSTREE 8888
+}
+
+vm_change_update_policy() {
+    policy=$1; shift
+    vm_ansible_inline <<EOF
+- shell: |
+    cp /usr/etc/rpm-ostreed.conf /etc
+    echo -e "[Daemon]\nAutomaticUpdatePolicy=$policy" > /etc/rpm-ostreed.conf
+    rpm-ostree reload
+EOF
 }
 
 # APIs to build up a history on the server. Rather than wasting time

tests/vmcheck/test-autoupdate-check.sh

@@ -55,20 +55,10 @@ echo "ok prep"
 # start it up again since we rebooted
 vm_start_httpd ostree_server $REMOTE_OSTREE 8888
 
-change_policy() {
-  policy=$1; shift
-  vm_ansible_inline <<EOF
-- shell: |
-    cp /usr/etc/rpm-ostreed.conf /etc
-    echo -e "[Daemon]\nAutomaticUpdatePolicy=$policy" > /etc/rpm-ostreed.conf
-    rpm-ostree reload
-EOF
-}
-
 vm_rpmostree cleanup -m
 
 # make sure that off means off
-change_policy off
+vm_change_update_policy off
 vm_rpmostree status | grep 'auto updates disabled'
 vm_rpmostree upgrade --trigger-automatic-update-policy > out.txt
 assert_file_has_content out.txt "Automatic updates are not enabled; exiting"
@@ -87,7 +77,7 @@ assert_file_has_content out.txt "No updates available."
 echo "ok --check/--preview no updates"
 
 # ok, let's test out check
-change_policy check
+vm_change_update_change_policy check
 vm_rpmostree status | grep 'auto updates enabled (check'
 
 # build an *older version* and check that we don't report an update
@@ -160,7 +150,7 @@ assert_output
 echo "ok --check/--preview layered pkgs check policy"
 
 # check that --check/--preview still works even with policy off
-change_policy off
+vm_change_update_policy off
 vm_rpmostree cleanup -m
 vm_cmd systemctl stop rpm-ostreed
 vm_rpmostree status | grep 'auto updates disabled'
@@ -170,7 +160,7 @@ assert_output
 echo "ok --check/--preview layered pkgs off policy"
 
 # ok now let's add ostree updates in the picture
-change_policy check
+vm_change_update_policy check
 vm_ostreeupdate_create v2
 vm_rpmostree upgrade --trigger-automatic-update-policy
 
@@ -221,7 +211,7 @@ vm_rpmostree upgrade --preview > out-verbose.txt
 assert_output2
 echo "ok --check/--preview base pkgs check policy"
 
-change_policy off
+vm_change_update_policy off
 vm_rpmostree cleanup -m
 vm_cmd systemctl stop rpm-ostreed
 vm_rpmostree upgrade --check > out.txt

tests/vmcheck/test-autoupdate-stage.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright (C) 2018 Jonathan Lebon
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. ${commondir}/libtest.sh
+. ${commondir}/libvm.sh
+
+set -x
+
+# Prepare an OSTree repo with updates
+vm_ostreeupdate_prepare_reboot
+
+vm_ostreeupdate_create v2
+
+vm_rpmostree cleanup -m
+
+vm_rpmostree status > status.txt
+assert_file_has_content status.txt 'auto updates disabled'
+vm_change_update_policy ex-stage
+vm_rpmostree status > status.txt
+assert_file_has_content_literal status.txt 'auto updates enabled (ex-stage'
+
+vm_rpmostree upgrade --trigger-automatic-update-policy
+vm_assert_status_jq ".deployments[1][\"booted\"]" \
+                    ".deployments[0][\"staged\"]" \
+                    ".deployments[0][\"version\"] == \"v2\""
+vm_rpmostree status -v > status.txt
+assert_file_has_content status.txt "Staged: yes"
+# And ensure that we have new content in /etc after staging
+vm_cmd echo new-content-in-etc \> /etc/somenewfile
+vm_reboot
+vm_assert_status_jq ".deployments[0][\"booted\"]" \
+                    ".deployments[0][\"staged\"]|not" \
+                    ".deployments[1][\"staged\"]|not" \
+                    ".deployments[0][\"version\"] == \"v2\""
+vm_cmd cat /etc/somenewfile > somenewfile.txt
+assert_file_has_content somenewfile.txt new-content-in-etc
+echo "ok autoupdate staged"

tests/vmcheck/test-upgrades.sh

@@ -25,19 +25,7 @@ set -euo pipefail
 set -x
 
 # Prepare an OSTree repo with updates
-vm_ostreeupdate_prepare
-
-vm_rpmostree rebase vmcheckmote:vmcheck
-vm_reboot
-vm_rpmostree cleanup -pr
-vm_assert_status_jq ".deployments[0][\"origin\"] == \"vmcheckmote:vmcheck\"" \
-                    ".deployments[0][\"booted\"]" \
-                    ".deployments[0][\"version\"] == \"v1\""
-vm_rpmostree status > status.txt
-assert_file_has_content_literal status.txt 'auto updates disabled'
-# start it up again since we rebooted
-vm_start_httpd ostree_server $REMOTE_OSTREE 8888
-echo "ok prep"
+vm_ostreeupdate_prepare_reboot
 
 # Test base ostree update
 vm_ostreeupdate_create v2