Merge 9977597ffb into 8aaea0c65d

Release 2024.10

configure.ac

@@ -1,7 +1,7 @@
 AC_PREREQ([2.63])
 dnl To perform a release, follow the instructions in `docs/CONTRIBUTING.md`.
 m4_define([year_version], [2024])
-m4_define([release_version], [10])
+m4_define([release_version], [11])
 m4_define([package_version], [year_version.release_version])
 AC_INIT([libostree], [package_version], [walters@verbum.org])
 is_release_build=no

man/ostree-prepare-root.xml

@@ -120,20 +120,25 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
             <varlistentry>
                 <term><varname>root.transient</varname></term>
                 <listitem><para>A boolean value; the default is <literal>false</literal>.
-                   If this is set to <literal>true</literal>, then the <literal>/</literal> filesystem will be a writable <literal>overlayfs</literal>,
-                   with the upper directory being a hidden directory (in the underlying system root filesystem) that will persist across reboots by default.
-                   However, changes will <emphasis>be discarded</emphasis> on OS updates!
+                   Setting this flag to <literal>true</literal> requires composefs (See <literal>composefs.enabled</literal>).
+                   When enabled, the root mount point <literal>/</literal> will be an overlayfs whose contents will be stored
+                   in a tmpfs, and hence discarded on OS upgrade or reboot.
                 </para>
                 <para>
-                   Enabling this option can be very useful for cases such as packages (dpkg/rpm/etc) that write content into <literal>/opt</literal>,
-                   particularly where they expect the target to be writable at runtime.  To make that work, ensure that your <literal>/opt</literal>
-                   directory is *not* a symlink to <literal>/var/opt</literal>, but is just an empty directory.
-                </para>
-                <para>
-                   Note the <literal>/usr</literal> mount point remains read-only by default.  This option is independent of <literal>etc.transient</literal> and <literal>sysroot.readonly</literal>;
+                   This option is independent of <literal>etc.transient</literal> and <literal>sysroot.readonly</literal>;
                    it is supported for example to have <literal>root.transient=true</literal> but <literal>etc.transient=false</literal> in which case changes to <literal>/etc</literal> continue
                    to persist across updates, with the default OSTree 3-way merge applied.
-                </para></listitem>
+                   Also related to persistence it is important to emphasize that <literal>/sysroot</literal> (the physical root filesystem) is still persistent
+                   by default; in-place OS upgrades can be applied.
+                </para>
+                <para>
+                  Enabling this option can make it significantly easier to adopt an image-based model in some circumstances.
+                  For example, if you have a configuration management system that is inspecting machine-specific state and
+                  e.g. dynamically installing packages or applying configuration, it can more easily be adapted to
+                  run on each boot, while still shifting a portion (or ideally most) image configuration to build time
+                  as part of the base image/commit.
+                </para>
+                </listitem>
             </varlistentry>
             <varlistentry>
                 <term><varname>composefs.enabled</varname></term>

src/libostree/ostree-deployment-private.h

@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include "ostree.h"
 #include "ostree-deployment.h"
 
 G_BEGIN_DECLS
@@ -54,9 +55,12 @@ struct _OstreeDeployment
   gboolean finalization_locked;
   char **overlay_initrds;
   char *overlay_initrds_id;
+  gchar *version;
+  gboolean version_is_cached;
 };
 
 void _ostree_deployment_set_bootcsum (OstreeDeployment *self, const char *bootcsum);
+const char *_ostree_deployment_get_version (OstreeDeployment *self, OstreeRepo *repo);
 
 void _ostree_deployment_set_overlay_initrds (OstreeDeployment *self, char **overlay_initrds);
 

src/libostree/ostree-deployment.c

@@ -85,6 +85,31 @@ ostree_deployment_get_bootserial (OstreeDeployment *self)
   return self->bootserial;
 }
 
+const char *
+_ostree_deployment_get_version (OstreeDeployment  *self,
+                                OstreeRepo        *repo)
+{
+  g_return_val_if_fail (repo != NULL, NULL);
+
+  if (!self->version_is_cached)
+    {
+      /* Try extracting a version for this deployment. */
+      const gchar *csum = ostree_deployment_get_csum (self);
+
+      g_autoptr(GVariant) variant = NULL;
+      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, csum,
+            &variant, NULL))
+        return NULL;
+
+      g_autoptr(GVariant) metadata = g_variant_get_child_value (variant, 0);
+      g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_VERSION, "s", &self->version);
+
+      self->version_is_cached = TRUE;
+    }
+
+  return self->version;
+}
+
 /**
  * ostree_deployment_get_bootconfig:
  * @self: Deployment
@@ -255,6 +280,8 @@ ostree_deployment_clone (OstreeDeployment *self)
   OstreeDeployment *ret = ostree_deployment_new (
       self->index, self->osname, self->csum, self->deployserial, self->bootcsum, self->bootserial);
 
+  ret->version = g_strdup (self->version);
+
   new_bootconfig = ostree_bootconfig_parser_clone (self->bootconfig);
   ostree_deployment_set_bootconfig (ret, new_bootconfig);
 
@@ -324,6 +351,7 @@ ostree_deployment_finalize (GObject *object)
   g_free (self->osname);
   g_free (self->csum);
   g_free (self->bootcsum);
+  g_free (self->version);
   g_clear_object (&self->bootconfig);
   g_clear_pointer (&self->origin, g_key_file_unref);
   g_strfreev (self->overlay_initrds);

src/libostree/ostree-sysroot-deploy.c

@@ -2036,24 +2036,9 @@ install_deployment_kernel (OstreeSysroot *sysroot, int new_bootversion,
   if (val == NULL)
     return glnx_throw (error, "No PRETTY_NAME or ID in /etc/os-release");
 
-  g_autofree char *deployment_version = NULL;
+  const gchar *deployment_version = NULL;
   if (repo)
-    {
-      /* Try extracting a version for this deployment. */
-      const char *csum = ostree_deployment_get_csum (deployment);
-      g_autoptr (GVariant) variant = NULL;
-      g_autoptr (GVariant) metadata = NULL;
-
-      /* XXX Copying ot_admin_checksum_version() + bits from
-       *     ot-admin-builtin-status.c.  Maybe this should be
-       *     public API in libostree? */
-      if (ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, csum, &variant, NULL))
-        {
-          metadata = g_variant_get_child_value (variant, 0);
-          (void)g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_VERSION, "s",
-                                  &deployment_version);
-        }
-    }
+    deployment_version = _ostree_deployment_get_version (deployment, repo);
 
   /* XXX The SYSLINUX bootloader backend actually parses the title string
    *     (specifically, it looks for the substring "(ostree"), so further
@@ -3840,6 +3825,11 @@ ostree_sysroot_stage_tree_with_options (OstreeSysroot *self, const char *osname,
     g_variant_builder_add (builder, "{sv}", "overlay-initrds",
                            g_variant_new_strv ((const char *const *)opts->overlay_initrds, -1));
 
+  /* Proxy across any flags */
+  if (opts && opts->finalization_flags)
+    g_variant_builder_add (builder, "{sv}", "write-deployment-flags",
+                           g_variant_new_uint32 ((guint32)opts->finalization_flags));
+
   const char *parent = dirname (strdupa (_OSTREE_SYSROOT_RUNSTATE_STAGED));
   if (!glnx_shutil_mkdir_p_at (AT_FDCWD, parent, 0755, cancellable, error))
     return FALSE;
@@ -4033,14 +4023,14 @@ _ostree_sysroot_finalize_staged_inner (OstreeSysroot *self, GCancellable *cancel
   staged->staged = FALSE;
   g_ptr_array_remove_index (self->deployments, 0);
 
-  /* TODO: Proxy across flags too?
-   *
-   * But note that we always use NO_CLEAN to avoid adding more latency at
+  OstreeSysrootSimpleWriteDeploymentFlags flags = 0;
+  g_variant_lookup (self->staged_deployment_data, "write-deployment-flags", "u", &flags);
+  /*
+   * Note that we always use NO_CLEAN to avoid adding more latency at
    * shutdown, and also because e.g. rpm-ostree wants to own the cleanup
    * process.
    */
-  OstreeSysrootSimpleWriteDeploymentFlags flags
-      = OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN;
+  flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN;
   if (!ostree_sysroot_simple_write_deployment (self, ostree_deployment_get_osname (staged), staged,
                                                merge_deployment, flags, cancellable, error))
     return FALSE;

src/libostree/ostree-sysroot.c

@@ -1905,6 +1905,9 @@ ostree_sysroot_init_osname (OstreeSysroot *self, const char *osname, GCancellabl
  * specified, then no cleanup will be performed after adding the
  * deployment. Make sure to call ostree_sysroot_cleanup() sometime
  * later, instead.
+ *
+ * If %OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PREVIOUS_VERSION is
+ * specified, then the previous version will not be garbage collected.
  */
 gboolean
 ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, const char *osname,
@@ -1920,6 +1923,8 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, const char *osna
       = (flags & OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PENDING) > 0;
   const gboolean retain_rollback
       = (flags & OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_ROLLBACK) > 0;
+  const gboolean retain_previous
+      = (flags & OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PREVIOUS_VERSION) > 0;
   gboolean retain = (flags & OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN) > 0;
 
   g_autoptr (GPtrArray) deployments = ostree_sysroot_get_deployments (sysroot);
@@ -1941,6 +1946,34 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, const char *osna
   if (!booted_deployment && !merge_deployment && (retain_pending || retain_rollback))
     retain = TRUE;
 
+  /* tracks current versioned deployment */
+  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
+  const gchar *new_version = _ostree_deployment_get_version (new_deployment, repo);
+
+  gboolean retained_previous_version = FALSE;
+
+  /* we never prune booted and merge deployments, so if they exist and are of a
+   * different version from `new_version`, they already fulfill the criteria of
+   * retaining the previous version */
+  if (booted_deployment)
+    {
+      const gchar *booted_version =
+        _ostree_deployment_get_version (booted_deployment, repo);
+      retained_previous_version = (g_strcmp0 (booted_version, new_version) != 0);
+    }
+
+  /* checking also that booted and merge are not the same although that's not a
+   * big deal since we cache the version now (though this will still work in
+   * the NULL case)
+   */
+  if (!retained_previous_version && merge_deployment &&
+      !ostree_deployment_equal (merge_deployment, booted_deployment))
+    {
+      const gchar *merge_version =
+        _ostree_deployment_get_version (merge_deployment, repo);
+      retained_previous_version = (g_strcmp0 (merge_version, new_version) != 0);
+    }
+
   /* tracks when we come across the booted deployment */
   gboolean before_booted = TRUE;
   gboolean before_merge = TRUE;
@@ -1962,6 +1995,13 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, const char *osna
        * deployments, fall back on merge deployment */
       const gboolean passed_crossover = booted_deployment ? !before_booted : !before_merge;
 
+      gboolean is_previous_version = FALSE;
+      if (passed_crossover && osname_matches && !retained_previous_version)
+        {
+          const gchar *version = _ostree_deployment_get_version (deployment, repo);
+          is_previous_version = (g_strcmp0 (version, new_version) != 0);
+        }
+
       /* Retain deployment if:
        *   - we're explicitly asked to, or
        *   - it's pinned
@@ -1974,6 +2014,15 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, const char *osna
           || (retain_pending && !passed_crossover) || (is_booted || is_merge)
           || (retain_rollback && passed_crossover))
         g_ptr_array_add (new_deployments, g_object_ref (deployment));
+      /*
+       *   - we're keeping the previous version deployment
+       */
+      else if (retain_previous && !retained_previous_version && is_previous_version)
+        {
+          g_ptr_array_add (new_deployments, g_object_ref (deployment));
+          /* Just keep one previous version */
+          retained_previous_version = TRUE;
+        }
 
       /* add right after booted/merge deployment */
       if (!added_new && passed_crossover)

src/libostree/ostree-sysroot.h

@@ -176,13 +176,17 @@ _OSTREE_PUBLIC
 gboolean ostree_sysroot_stage_overlay_initrd (OstreeSysroot *self, int fd, char **out_checksum,
                                               GCancellable *cancellable, GError **error);
 
+/**
+ * finalization_flags: only used in the staging path
+ */
 typedef struct
 {
   /* If set to true, then this deployment will be staged but "locked" and not automatically applied
    * on reboot. */
   gboolean locked;
   gboolean unused_bools[7];
-  int unused_ints[8];
+  int finalization_flags;
+  int unused_ints[7];
   char **override_kernel_argv;
   char **overlay_initrds;
   gpointer unused_ptrs[6];
@@ -259,6 +263,7 @@ typedef enum
   OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN = (1 << 2),
   OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PENDING = (1 << 3),
   OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_ROLLBACK = (1 << 4),
+  OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PREVIOUS_VERSION = (1 << 5),
 } OstreeSysrootSimpleWriteDeploymentFlags;
 
 _OSTREE_PUBLIC

src/ostree/ot-admin-builtin-deploy.c

@@ -35,6 +35,7 @@ static gboolean opt_stage;
 static gboolean opt_lock_finalization;
 static gboolean opt_retain_pending;
 static gboolean opt_retain_rollback;
+static gboolean opt_retain_previous;
 static gboolean opt_not_as_default;
 static gboolean opt_no_prune;
 static gboolean opt_no_merge;
@@ -65,6 +66,8 @@ static GOptionEntry options[] = {
     "Do not delete pending deployments", NULL },
   { "retain-rollback", 0, 0, G_OPTION_ARG_NONE, &opt_retain_rollback,
     "Do not delete rollback deployments", NULL },
+  { "retain-previous-version", 0, 0, G_OPTION_ARG_NONE, &opt_retain_previous,
+    "Do not delete previous deployment if version differs from new deployment", NULL },
   { "not-as-default", 0, 0, G_OPTION_ARG_NONE, &opt_not_as_default,
     "Append rather than prepend new deployment", NULL },
   { "karg-proc-cmdline", 0, 0, G_OPTION_ARG_NONE, &opt_kernel_proc_cmdline,
@@ -239,6 +242,7 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat
     .locked = opt_lock_finalization,
     .override_kernel_argv = kargs_strv,
     .overlay_initrds = overlay_initrd_chksums ? (char **)overlay_initrd_chksums->pdata : NULL,
+    .finalization_flags = opt_retain_previous ? OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PREVIOUS_VERSION : 0,
   };
 
   g_autoptr (OstreeDeployment) new_deployment = NULL;
@@ -265,7 +269,7 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat
                                             _OSTREE_SYSROOT_RUNSTATE_STAGED_LOCKED);
         }
       /* use old API if we can to exercise it in CI */
-      if (!(overlay_initrd_chksums || opt_lock_finalization))
+      if (!(overlay_initrd_chksums || opt_lock_finalization || opt_retain_previous))
         {
           if (!ostree_sysroot_stage_tree (sysroot, opt_osname, revision, origin, merge_deployment,
                                           kargs_strv, &new_deployment, cancellable, error))
@@ -308,6 +312,8 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat
             flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PENDING;
           if (opt_retain_rollback)
             flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_ROLLBACK;
+          if (opt_retain_previous)
+            flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PREVIOUS_VERSION;
         }
 
       if (opt_not_as_default)

tests/kolainst/destructive/staged-deploy.sh

@@ -46,7 +46,7 @@ EOF
     # xref https://github.com/coreos/coreos-assembler/pull/2814
     systemctl mask --now zincati
     # Create a synthetic commit for upgrade
-    ostree commit --no-bindings --parent="${commit}" -b staged-deploy -I --consume t
+    ostree commit --no-bindings --parent="${commit}" -b staged-deploy -I --consume t --add-metadata-string=version=foobar
     newcommit=$(ostree rev-parse staged-deploy)
     orig_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy)
     systemctl show -p SubState ostree-finalize-staged.path | grep -q waiting
@@ -169,8 +169,46 @@ EOF
     ostree admin undeploy 1
     echo "ok staged retained"
 
+    # Deploy a new version
+    commit=${host_commit}
+    ostree checkout -H ${commit} t
+    ostree commit --no-bindings --parent="${commit}" -b same-version -I --consume t --add-metadata-string=version=foobaz
+    ostree admin deploy same-version --retain-previous-version
+
     # Cleanup refs
-    ostree refs --delete staged-deploy nonstaged-deploy
+    ostree refs --delete staged-deploy nonstaged-deploy same-version
+    echo "ok cleanup refs"
+
+    /tmp/autopkgtest-reboot "3"
+    ;;
+  "3")
+    # Check currently deployed versions
+    rpm-ostree status
+
+    # Make a new commit with the same version as the previous reboot
+    commit=$(rpmostree_query_json '.deployments[0].checksum')
+    cd /ostree/repo/tmp
+    ostree checkout -H ${commit} t
+    ostree commit --no-bindings --parent="${commit}" -b same-version-again -I --consume t --add-metadata-string=version=foobaz
+    ostree admin deploy same-version-again --retain-previous-version
+
+    # Check that previous version was kept
+    ostree admin status > status.txt
+    test $(grep -Fce 'Version: ' status.txt) = 3
+    echo "ok previous version retained"
+
+    # Check also for the staging path
+    rpm-ostree cleanup -p
+    ostree admin deploy --stage same-version-again --retain-previous-version
+    ostree admin finalize-staged
+
+    # Check that previous version was kept
+    ostree admin status > status.txt
+    test $(grep -Fce 'Version: ' status.txt) = 3
+    echo "ok previous version retained during stage"
+
+    # Cleanup refs
+    ostree refs --delete same-version-again
     echo "ok cleanup refs"
 
     # Now finally, try breaking staged updates and verify that ostree-boot-complete fails on the next boot
@@ -193,9 +231,9 @@ WantedBy=multi-user.target
 EOF
     systemctl enable hackaround-cosa-systemd-unit-checks.service
 
-    /tmp/autopkgtest-reboot "3"
+    /tmp/autopkgtest-reboot "4"
     ;;
-  "3") 
+  "4")
     assert_file_has_content /run/ostree-boot-complete-status.txt 'error: ostree-finalize-staged.service failed on previous boot.*Operation not permitted'
     echo "ok boot-complete.service"
     ;;