Merge dd9b8700ff into 8aaea0c65d

Release 2024.10

configure.ac

@@ -1,7 +1,7 @@
 AC_PREREQ([2.63])
 dnl To perform a release, follow the instructions in `docs/CONTRIBUTING.md`.
 m4_define([year_version], [2024])
-m4_define([release_version], [10])
+m4_define([release_version], [11])
 m4_define([package_version], [year_version.release_version])
 AC_INIT([libostree], [package_version], [walters@verbum.org])
 is_release_build=no

man/ostree-commit.xml

@@ -177,6 +177,15 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--quote-devices</option></term>
+                <listitem><para>
+                    By default, ostree rejects block and character devices. This option instead "quotes" them
+                    as regular files. In order to be processed back into block and character devices,
+                    the corresponding <literal>--unquote-devices</literal> must be passed to <literal>ostree checkout</literal>.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--no-xattrs</option></term>
                 <listitem><para>

man/ostree-prepare-root.xml

@@ -120,20 +120,25 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
             <varlistentry>
                 <term><varname>root.transient</varname></term>
                 <listitem><para>A boolean value; the default is <literal>false</literal>.
-                   If this is set to <literal>true</literal>, then the <literal>/</literal> filesystem will be a writable <literal>overlayfs</literal>,
+                   Setting this flag to <literal>true</literal> requires composefs (See <literal>composefs.enabled</literal>).
-                   with the upper directory being a hidden directory (in the underlying system root filesystem) that will persist across reboots by default.
+                   When enabled, the root mount point <literal>/</literal> will be an overlayfs whose contents will be stored
-                   However, changes will <emphasis>be discarded</emphasis> on OS updates!
+                   in a tmpfs, and hence discarded on OS upgrade or reboot.
                 </para>
                 <para>
-                   Enabling this option can be very useful for cases such as packages (dpkg/rpm/etc) that write content into <literal>/opt</literal>,
+                   This option is independent of <literal>etc.transient</literal> and <literal>sysroot.readonly</literal>;
-                   particularly where they expect the target to be writable at runtime.  To make that work, ensure that your <literal>/opt</literal>
-                   directory is *not* a symlink to <literal>/var/opt</literal>, but is just an empty directory.
-                </para>
-                <para>
-                   Note the <literal>/usr</literal> mount point remains read-only by default.  This option is independent of <literal>etc.transient</literal> and <literal>sysroot.readonly</literal>;
                    it is supported for example to have <literal>root.transient=true</literal> but <literal>etc.transient=false</literal> in which case changes to <literal>/etc</literal> continue
                    to persist across updates, with the default OSTree 3-way merge applied.
-                </para></listitem>
+                   Also related to persistence it is important to emphasize that <literal>/sysroot</literal> (the physical root filesystem) is still persistent
+                   by default; in-place OS upgrades can be applied.
+                </para>
+                <para>
+                  Enabling this option can make it significantly easier to adopt an image-based model in some circumstances.
+                  For example, if you have a configuration management system that is inspecting machine-specific state and
+                  e.g. dynamically installing packages or applying configuration, it can more easily be adapted to
+                  run on each boot, while still shifting a portion (or ideally most) image configuration to build time
+                  as part of the base image/commit.
+                </para>
+                </listitem>
             </varlistentry>
             <varlistentry>
                 <term><varname>composefs.enabled</varname></term>

src/libostree/ostree-core-private.h

@@ -78,6 +78,20 @@ G_BEGIN_DECLS
  */
 #define _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT G_VARIANT_TYPE ("(tuuuusa(ayay))")
 
+// ostree doesn't have native support for devices. Whiteouts in overlayfs
+// are a 0:0 character device, and in some cases people are copying docker/podman
+// style overlayfs container storage directly into ostree commits. This
+// adds special support for "quoting" the whiteout so it just appears as a regular
+// file in the ostree commit, but can be converted back into a character device
+// on checkout.
+#define OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh."
+// Filename prefix to signify a character or block device. This
+// is not supported natively by ostree (because there is no reason
+// to ship devices in images). But because OCI supports it, and in
+// some cases one wants to map OCI to ostree, we have support for
+// "quoting" them.
+#define OSTREE_QUOTED_DEVICE_PREFIX ".ostree-quoted-device."
+
 GBytes *_ostree_file_header_new (GFileInfo *file_info, GVariant *xattrs);
 
 GBytes *_ostree_zlib_file_header_new (GFileInfo *file_info, GVariant *xattrs);
@@ -92,6 +106,9 @@ gboolean _ostree_stbuf_equal (struct stat *stbuf_a, struct stat *stbuf_b);
 GFileInfo *_ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid);
 gboolean _ostree_validate_structureof_xattrs (GVariant *xattrs, GError **error);
 
+gboolean _ostree_parse_quoted_device (const char *name, guint32 src_mode, const char **out_name, guint32 *out_mode,
+                                      dev_t *out_dev, GError **error);
+
 static inline void
 _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf)
 {

src/libostree/ostree-core.c

@@ -33,6 +33,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/types.h>
+#include <sys/sysmacros.h>
 
 /* Generic ABI checks */
 G_STATIC_ASSERT (OSTREE_REPO_MODE_BARE == 0);
@@ -2383,6 +2384,70 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta, GError **error)
   return TRUE;
 }
 
+gboolean
+_ostree_parse_quoted_device (const char *name, guint32 src_mode, const char **out_name, guint32 *out_mode, dev_t *out_dev,
+                             GError **error)
+{
+  // Ensure we start with the quoted device prefix
+  const char *s = name;
+  const char *p = strchr (s, '.');
+  if (!p)
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  if (strncmp (s, OSTREE_QUOTED_DEVICE_PREFIX, p - name) != 0)
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  s += strlen (OSTREE_QUOTED_DEVICE_PREFIX);
+  g_assert (out_name);
+  *out_name = s;
+
+  // The input mode is the same as the source, but without the format bits
+  guint32 ret_mode = (src_mode & ~S_IFMT);
+
+  // Parse the mode
+  s++;
+  switch (*s)
+    {
+    case 'b':
+      ret_mode |= S_IFBLK;
+      break;
+    case 'c':
+      ret_mode |= S_IFCHR;
+      break;
+    case 'p':
+      ret_mode |= S_IFIFO;
+      break;
+    default:
+      return glnx_throw (error, "Invalid quoted device: %s", name);
+    }
+  s++;
+  if (*s != '.')
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  s++;
+  s = strchr (s, '.');
+  if (!s)
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  s++;
+  char *endptr;
+  unsigned int major, minor;
+  major = (unsigned int)g_ascii_strtoull (s, &endptr, 10);
+  if (errno == ERANGE)
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  s = endptr;
+  if (*s != '.')
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  s++;
+  minor = (unsigned int)g_ascii_strtoull (s, &endptr, 10);
+  if (errno == ERANGE)
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  g_assert (endptr);
+  if (*endptr != '\0')
+    return glnx_throw (error, "Invalid quoted device: %s", name);
+  g_assert (ret_mode);
+  *out_mode = ret_mode;
+  g_assert (out_dev);
+  *out_dev = makedev (major, minor);
+  return TRUE;
+}
+
 /**
  * ostree_commit_get_parent:
  * @commit_variant: Variant of type %OSTREE_OBJECT_TYPE_COMMIT

src/libostree/ostree-repo-checkout.c

@@ -35,14 +35,6 @@
 #define WHITEOUT_PREFIX ".wh."
 #define OPAQUE_WHITEOUT_NAME ".wh..wh..opq"
 
-// ostree doesn't have native support for devices. Whiteouts in overlayfs
-// are a 0:0 character device, and in some cases people are copying docker/podman
-// style overlayfs container storage directly into ostree commits. This
-// adds special support for "quoting" the whiteout so it just appears as a regular
-// file in the ostree commit, but can be converted back into a character device
-// on checkout.
-#define OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh."
-
 /* Per-checkout call state/caching */
 typedef struct
 {
@@ -716,6 +708,9 @@ checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, Ch
   const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0);
   const gboolean is_whiteout = (!is_symlink && options->process_whiteouts
                                 && g_str_has_prefix (destination_name, WHITEOUT_PREFIX));
+  const gboolean is_quoted_device
+      = (!is_symlink && options->unquote_devices
+         && g_str_has_prefix (destination_name, OSTREE_QUOTED_DEVICE_PREFIX));
   const gboolean is_overlayfs_whiteout
       = (!is_symlink
          && g_str_has_prefix (destination_name, OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX));
@@ -740,6 +735,16 @@ checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, Ch
 
       need_copy = FALSE;
     }
+  else if (is_quoted_device)
+    {
+      const char *devname;
+      dev_t dev;
+      guint32 mode;
+      if (!_ostree_parse_quoted_device (destination_name, source_mode, &devname, &mode, &dev, error))
+        return FALSE;
+      if (mknodat (destination_dfd, devname, (mode_t)mode, dev) < 0)
+        return glnx_throw_errno_prefix (error, "mknodat");
+    }
   else if (is_overlayfs_whiteout && options->process_passthrough_whiteouts)
     {
       const char *name = destination_name + (sizeof (OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX) - 1);
@@ -1469,6 +1474,9 @@ canonicalize_options (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options)
   /* Force USER mode for BARE_USER_ONLY always - nothing else makes sense */
   if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY)
     options->mode = OSTREE_REPO_CHECKOUT_MODE_USER;
+
+  if (options->unquote_devices)
+    options->process_whiteouts = TRUE;
 }
 
 /**

src/libostree/ostree-repo-commit.c

@@ -3451,6 +3451,202 @@ write_dir_entry_to_mtree_internal (OstreeRepo *self, OstreeRepoFile *repo_dir,
   return TRUE;
 }
 
+static gboolean
+write_quoted_device (OstreeRepo *self, OstreeRepoFile *repo_dir,
+                                 GFileEnumerator *dir_enum, GLnxDirFdIterator *dfd_iter,
+                                 WriteDirContentFlags writeflags, GFileInfo *child_info,
+                                 OstreeMutableTree *mtree, OstreeRepoCommitModifier *modifier,
+                                 GPtrArray *path, GCancellable *cancellable, GError **error)
+{
+  g_assert (dir_enum != NULL || dfd_iter != NULL);
+
+  GFileType file_type = g_file_info_get_file_type (child_info);
+  const char *name = g_file_info_get_name (child_info);
+
+  /* Load flags into boolean constants for ease of readability (we also need to
+   * NULL-check modifier)
+   */
+  const gboolean canonical_permissions
+      = self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY
+        || (modifier
+            && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS));
+  const gboolean devino_canonical
+      = modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL);
+  /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even
+   * more complexity in this function, and it'd mostly only be useful when
+   * operating on local filesystems anyways.
+   */
+  const gboolean delete_after_commit
+      = dfd_iter && modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+
+  /* Build the full path which we need for callbacks */
+  g_ptr_array_add (path, (char *)name);
+  g_autofree char *child_relpath = ptrarray_path_join (path);
+
+  /* Call the filter */
+  g_autoptr (GFileInfo) modified_info = NULL;
+  OstreeRepoCommitFilterResult filter_result = _ostree_repo_commit_modifier_apply (
+      self, modifier, child_relpath, child_info, &modified_info);
+  const gboolean child_info_was_modified = !_ostree_gfileinfo_equal (child_info, modified_info);
+
+  if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW)
+    {
+      g_ptr_array_remove_index (path, path->len - 1);
+      if (delete_after_commit)
+        {
+          g_assert (dfd_iter);
+          if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
+            return FALSE;
+        }
+      /* Note: early return */
+      return TRUE;
+    }
+
+  guint32 src_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode")';'
+  switch (file_type)
+    {
+    case G_FILE_TYPE_SYMBOLIC_LINK:
+    case G_FILE_TYPE_REGULAR:
+      break;
+    default:
+      return glnx_throw (error, "Unsupported file type for file: '%s'", child_relpath);
+    }
+
+  g_autoptr (GFile) child = NULL;
+  if (dir_enum != NULL)
+    child = g_file_enumerator_get_child (dir_enum, child_info);
+
+  /* Our filters have passed, etc.; now we prepare to write the content object */
+  glnx_autofd int file_input_fd = -1;
+
+  /* Open the file now, since it's better for reading xattrs
+   * rather than using the /proc/self/fd links.
+   *
+   * TODO: Do this lazily, since for e.g. bare-user-only repos
+   * we don't have xattrs and don't need to open every file
+   * for things that have devino cache hits.
+   */
+  if (file_type == G_FILE_TYPE_REGULAR && dfd_iter != NULL)
+    {
+      if (!glnx_openat_rdonly (dfd_iter->fd, name, FALSE, &file_input_fd, error))
+        return FALSE;
+    }
+
+  g_autoptr (GVariant) xattrs = NULL;
+  gboolean xattrs_were_modified;
+  if (dir_enum != NULL)
+    {
+      if (!get_final_xattrs (self, modifier, child_relpath, child_info, child, -1, name,
+                             source_xattrs, &xattrs, &xattrs_were_modified, cancellable, error))
+        return FALSE;
+    }
+  else
+    {
+      /* These contortions are basically so we use glnx_fd_get_all_xattrs()
+       * for regfiles, and glnx_dfd_name_get_all_xattrs() for symlinks.
+       */
+      int xattr_fd_arg = (file_input_fd != -1) ? file_input_fd : dfd_iter->fd;
+      const char *xattr_path_arg = (file_input_fd != -1) ? NULL : name;
+      if (!get_final_xattrs (self, modifier, child_relpath, child_info, child, xattr_fd_arg,
+                             xattr_path_arg, source_xattrs, &xattrs, &xattrs_were_modified,
+                             cancellable, error))
+        return FALSE;
+    }
+
+  /* Used below to see whether we can do a fast path commit */
+  const gboolean modified_file_meta = child_info_was_modified || xattrs_were_modified;
+
+  /* A big prerequisite list of conditions for whether or not we can
+   * "adopt", i.e. just checksum and rename() into place
+   */
+  const gboolean can_adopt_basic = file_type == G_FILE_TYPE_REGULAR && dfd_iter != NULL
+                                   && delete_after_commit
+                                   && ((writeflags & WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT) > 0);
+  gboolean can_adopt = can_adopt_basic;
+  /* If basic prerquisites are met, check repo mode specific ones */
+  if (can_adopt)
+    {
+      /* For bare repos, we could actually chown/reset the xattrs, but let's
+       * do the basic optimizations here first.
+       */
+      if (self->mode == OSTREE_REPO_MODE_BARE)
+        can_adopt = !modified_file_meta;
+      else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
+        can_adopt = canonical_permissions;
+      else
+        /* This covers bare-user and archive.  See comments in adopt_and_commit_regfile()
+         * for notes on adding bare-user later here.
+         */
+        can_adopt = FALSE;
+    }
+  gboolean did_adopt = FALSE;
+
+  /* The very fast path - we have a devino cache hit, nothing to write */
+  if (loose_checksum && !modified_file_meta)
+    {
+      if (!ostree_mutable_tree_replace_file (mtree, name, loose_checksum, error))
+        return FALSE;
+
+      g_mutex_lock (&self->txn_lock);
+      self->txn.stats.devino_cache_hits++;
+      g_mutex_unlock (&self->txn_lock);
+    }
+  /* Next fast path - we can "adopt" the file */
+  else if (can_adopt)
+    {
+      char checksum[OSTREE_SHA256_STRING_LEN + 1];
+      if (!adopt_and_commit_regfile (self, dfd_iter->fd, name, modified_info, xattrs, checksum,
+                                     cancellable, error))
+        return FALSE;
+      if (!ostree_mutable_tree_replace_file (mtree, name, checksum, error))
+        return FALSE;
+      did_adopt = TRUE;
+    }
+  else
+    {
+      g_autoptr (GInputStream) file_input = NULL;
+
+      if (file_type == G_FILE_TYPE_REGULAR)
+        {
+          if (dir_enum != NULL)
+            {
+              g_assert (child != NULL);
+              file_input = (GInputStream *)g_file_read (child, cancellable, error);
+              if (!file_input)
+                return FALSE;
+            }
+          else
+            {
+              /* We already opened the fd above */
+              file_input = g_unix_input_stream_new (file_input_fd, FALSE);
+            }
+        }
+
+      g_autofree guchar *child_file_csum = NULL;
+      if (!write_content_object (self, NULL, file_input, modified_info, xattrs, &child_file_csum,
+                                 cancellable, error))
+        return FALSE;
+
+      char tmp_checksum[OSTREE_SHA256_STRING_LEN + 1];
+      ostree_checksum_inplace_from_bytes (child_file_csum, tmp_checksum);
+      if (!ostree_mutable_tree_replace_file (mtree, name, tmp_checksum, error))
+        return FALSE;
+    }
+
+  /* Process delete_after_commit. In the adoption case though, we already
+   * took ownership of the file above, usually via a renameat().
+   */
+  if (delete_after_commit && !did_adopt)
+    {
+      if (!glnx_unlinkat (dfd_iter->fd, name, 0, error))
+        return FALSE;
+    }
+
+  g_ptr_array_remove_index (path, path->len - 1);
+
+  return TRUE;
+}
+
 /* Given either a dir_enum or a dfd_iter, writes a non-dir (regfile/symlink) to
  * the mtree.
  */
@@ -3890,6 +4086,14 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, GLnxDirFdIterator *src_dfd_i
                                          error))
             return FALSE;
         }
+      else if (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES)
+        {
+          if (!write_quoted_device (self, NULL, NULL, src_dfd_iter, flags, child_info, mtree,
+                                    modifier, path, cancellable, error))
+            return FALSE;
+          // Note we skip over the code below
+          continue;
+        }
       else
         {
           return glnx_throw (error, "Not a regular file or symlink: %s", dent->d_name);

src/libostree/ostree-repo.h

@@ -519,6 +519,9 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo *repo
  * modifier filters (non-directories only); Since: 2017.14
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SELINUX_LABEL_V1: For SELinux and other systems, label
  * /usr/etc as if it was /etc.
+ * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES: Instead of erroring out on block/character
+ * devices, "quote" them as regular files that can optionally be unpacked back into native devices.
+ * Since: 2024.9
  *
  * Flags modifying commit behavior. In bare-user-only mode,
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS and
@@ -535,6 +538,7 @@ typedef enum
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME = (1 << 4),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL = (1 << 5),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SELINUX_LABEL_V1 = (1 << 6),
+  OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES = (1 << 7),
 } OstreeRepoCommitModifierFlags;
 
 /**
@@ -802,12 +806,13 @@ typedef struct
   gboolean enable_uncompressed_cache; /* Deprecated */
   gboolean enable_fsync;              /* Deprecated */
   gboolean process_whiteouts;
+  gboolean unquote_devices; /* Since: 2024.9 */
   gboolean no_copy_fallback;
   gboolean force_copy;           /* Since: 2017.6 */
   gboolean bareuseronly_dirs;    /* Since: 2017.7 */
   gboolean force_copy_zerosized; /* Since: 2018.9 */
   gboolean process_passthrough_whiteouts;
-  gboolean unused_bools[3];
+  gboolean unused_bools[2];
   /* 3 byte hole on 64 bit */
 
   const char *subpath;

src/ostree/ot-builtin-commit.c

@@ -62,6 +62,7 @@ static char *opt_base;
 static char **opt_trees;
 static gint opt_owner_uid = -1;
 static gint opt_owner_gid = -1;
+static gboolean opt_quote_devices;
 static gboolean opt_table_output;
 #ifndef OSTREE_DISABLE_GPGME
 static char **opt_gpg_key_ids;
@@ -124,6 +125,8 @@ static GOptionEntry options[] = {
   { "owner-gid", 0, 0, G_OPTION_ARG_INT, &opt_owner_gid, "Set file ownership group id", "GID" },
   { "canonical-permissions", 0, 0, G_OPTION_ARG_NONE, &opt_canonical_permissions,
     "Canonicalize permissions in the same way bare-user does for hardlinked files", NULL },
+  { "quote-devices", 0, 0, G_OPTION_ARG_NONE, &opt_quote_devices,
+    "Instead of erroring out on block/character devices, \"quote\" them as regular files", NULL },
   { "bootable", 0, 0, G_OPTION_ARG_NONE, &opt_bootable,
     "Flag this commit as a bootable OSTree (e.g. contains a Linux kernel)", NULL },
   { "mode-ro-executables", 0, 0, G_OPTION_ARG_NONE, &opt_ro_executables,
@@ -601,6 +604,8 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS;
   if (opt_consume)
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME;
+  if (opt_quote_devices)
+    flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES;
   switch (opt_selinux_labeling_epoch)
     {
     case 0: