checkout: Add API to directly checkout composefs

We were missing the simple, obvious API and CLI to go
from ostree commit -> composefs.

Internally, we had `ostree_repo_checkout_composefs`
with the right "shape" mostly, except it had more code
in the deploy path to turn that into a composefs.

Add a straightforward public API that does what
the deploy code did before, and then the old
API becomes an explicitly internal helper with an `_`
prefix.

Goals:

- Lead towards a composefs-oriented future
- This makes the composefs logic more testable directly

Signed-off-by: Colin Walters <walters@verbum.org>

Makefile-libostree.am

@@ -176,7 +176,7 @@ symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
 
 # Uncomment this include when adding new development symbols.
 if BUILDOPT_IS_DEVEL_BUILD
-#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
 endif
 
 # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html

apidoc/ostree-sections.txt

@@ -435,6 +435,7 @@ OstreeRepoCheckoutOverwriteMode
 ostree_repo_checkout_tree
 ostree_repo_checkout_tree_at
 ostree_repo_checkout_at
+ostree_repo_checkout_composefs
 ostree_repo_checkout_gc
 ostree_repo_read_commit
 OstreeRepoListObjectsFlags

src/libostree/libostree-devel.sym

@@ -20,6 +20,11 @@
    - uncomment the include in Makefile-libostree.am
 */
 
+LIBOSTREE_2024.7 {
+global:
+  ostree_repo_checkout_composefs;
+} LIBOSTREE_2023.8;
+
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste
  * source.  Replace $LASTSTABLE with the last stable version, and $NEWVERSION

src/libostree/ostree-repo-checkout.c

@@ -1232,6 +1232,106 @@ checkout_tree_at_recurse (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options
   return TRUE;
 }
 
+#ifdef HAVE_COMPOSEFS
+static gboolean
+compare_verity_digests (GVariant *metadata_composefs, const guchar *fsverity_digest, GError **error)
+{
+  const guchar *expected_digest;
+
+  if (metadata_composefs == NULL)
+    return TRUE;
+
+  if (g_variant_n_children (metadata_composefs) != OSTREE_SHA256_DIGEST_LEN)
+    return glnx_throw (error, "Expected composefs fs-verity in metadata has the wrong size");
+
+  expected_digest = g_variant_get_data (metadata_composefs);
+  if (memcmp (fsverity_digest, expected_digest, OSTREE_SHA256_DIGEST_LEN) != 0)
+    {
+      char actual_checksum[OSTREE_SHA256_STRING_LEN + 1];
+      char expected_checksum[OSTREE_SHA256_STRING_LEN + 1];
+
+      ostree_checksum_inplace_from_bytes (fsverity_digest, actual_checksum);
+      ostree_checksum_inplace_from_bytes (expected_digest, expected_checksum);
+
+      return glnx_throw (error,
+                         "Generated composefs image digest (%s) doesn't match expected digest (%s)",
+                         actual_checksum, expected_checksum);
+    }
+
+  return TRUE;
+}
+
+#endif
+
+/**
+ * ostree_repo_checkout_composefs:
+ * @self: A repo
+ * @options: (nullable): Future expansion space; must currently be %NULL
+ * @destination_dfd: Parent directory fd
+ * @destination_path: Filename
+ * @checksum: OStree commit digest
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Create a composefs filesystem metadata blob from an OSTree commit.
+ */
+gboolean
+ostree_repo_checkout_composefs (OstreeRepo *self, GVariant *options, int destination_dfd,
+                                const char *destination_path, const char *checksum,
+                                GCancellable *cancellable, GError **error)
+{
+#ifdef HAVE_COMPOSEFS
+  /* Force this for now */
+  g_assert (options == NULL);
+
+  g_auto (GLnxTmpfile) tmpf = {
+    0,
+  };
+  if (!glnx_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, &tmpf, error))
+    return FALSE;
+
+  g_autoptr (GVariant) commit_variant = NULL;
+  if (!ostree_repo_load_commit (self, checksum, &commit_variant, NULL, error))
+    return FALSE;
+
+  g_autoptr (GVariant) metadata = g_variant_get_child_value (commit_variant, 0);
+  g_autoptr (GVariant) metadata_composefs = g_variant_lookup_value (
+      metadata, OSTREE_COMPOSEFS_DIGEST_KEY_V0, G_VARIANT_TYPE_BYTESTRING);
+
+  g_autoptr (GFile) commit_root = NULL;
+  if (!ostree_repo_read_commit (self, checksum, &commit_root, NULL, cancellable, error))
+    return FALSE;
+
+  g_autoptr (OstreeComposefsTarget) target = ostree_composefs_target_new ();
+
+  if (!_ostree_repo_checkout_composefs (self, target, (OstreeRepoFile *)commit_root, cancellable,
+                                        error))
+    return FALSE;
+
+  g_autofree guchar *fsverity_digest = NULL;
+  if (!ostree_composefs_target_write (target, tmpf.fd, &fsverity_digest, cancellable, error))
+    return FALSE;
+
+  /* If the commit specified a composefs digest, verify it */
+  if (!compare_verity_digests (metadata_composefs, fsverity_digest, error))
+    return FALSE;
+
+  if (!glnx_fchmod (tmpf.fd, 0644, error))
+    return FALSE;
+
+  if (!_ostree_tmpf_fsverity (self, &tmpf, NULL, error))
+    return FALSE;
+
+  if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, destination_dfd, destination_path,
+                             error))
+    return FALSE;
+
+  return TRUE;
+#else
+  return composefs_not_supported (error);
+#endif
+}
+
 /* Begin a checkout process */
 static gboolean
 checkout_tree_at (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options, int destination_parent_fd,

src/libostree/ostree-repo-composefs.c

@@ -180,14 +180,6 @@ _composefs_write_cb (void *file, void *buf, size_t len)
 
 #else /* HAVE_COMPOSEFS */
 
-static gboolean
-composefs_not_supported (GError **error)
-{
-  g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
-               "composefs is not supported in this ostree build");
-  return FALSE;
-}
-
 #endif
 
 /**
@@ -520,7 +512,7 @@ ensure_lcfs_dir (struct lcfs_node_s *parent, const char *name, GError **error)
 #endif /* HAVE_COMPOSEFS */
 
 /**
- * ostree_repo_checkout_composefs:
+ * _ostree_repo_checkout_composefs:
  * @self: Repo
  * @target: A target for the checkout
  * @source: Source tree
@@ -538,8 +530,8 @@ ensure_lcfs_dir (struct lcfs_node_s *parent, const char *name, GError **error)
  * Returns: %TRUE on success, %FALSE on failure
  */
 gboolean
-ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
+_ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
-                                OstreeRepoFile *source, GCancellable *cancellable, GError **error)
+                                 OstreeRepoFile *source, GCancellable *cancellable, GError **error)
 {
 #ifdef HAVE_COMPOSEFS
   GLNX_AUTO_PREFIX_ERROR ("Checking out composefs", error);
@@ -601,7 +593,7 @@ ostree_repo_commit_add_composefs_metadata (OstreeRepo *self, guint format_versio
 
   g_autoptr (OstreeComposefsTarget) target = ostree_composefs_target_new ();
 
-  if (!ostree_repo_checkout_composefs (self, target, repo_root, cancellable, error))
+  if (!_ostree_repo_checkout_composefs (self, target, repo_root, cancellable, error))
     return FALSE;
 
   g_autofree guchar *fsverity_digest = NULL;

src/libostree/ostree-repo-private.h

@@ -473,9 +473,16 @@ gboolean ostree_composefs_target_write (OstreeComposefsTarget *target, int fd,
                                         guchar **out_fsverity_digest, GCancellable *cancellable,
                                         GError **error);
 
-gboolean ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
+gboolean _ostree_repo_checkout_composefs (OstreeRepo *self, OstreeComposefsTarget *target,
-                                         OstreeRepoFile *source, GCancellable *cancellable,
+                                          OstreeRepoFile *source, GCancellable *cancellable,
-                                         GError **error);
+                                          GError **error);
+static inline gboolean
+composefs_not_supported (GError **error)
+{
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+               "composefs is not supported in this ostree build");
+  return FALSE;
+}
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeComposefsTarget, ostree_composefs_target_unref)
 

src/libostree/ostree-repo.h

@@ -840,6 +840,11 @@ gboolean ostree_repo_checkout_at (OstreeRepo *self, OstreeRepoCheckoutAtOptions
                                   int destination_dfd, const char *destination_path,
                                   const char *commit, GCancellable *cancellable, GError **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_checkout_composefs (OstreeRepo *self, GVariant *options, int destination_dfd,
+                                         const char *destination_path, const char *checksum,
+                                         GCancellable *cancellable, GError **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_checkout_gc (OstreeRepo *self, GCancellable *cancellable, GError **error);
 

src/libostree/ostree-sysroot-deploy.c

@@ -600,37 +600,6 @@ merge_configuration_from (OstreeSysroot *sysroot, OstreeDeployment *merge_deploy
   return TRUE;
 }
 
-#ifdef HAVE_COMPOSEFS
-static gboolean
-compare_verity_digests (GVariant *metadata_composefs, const guchar *fsverity_digest, GError **error)
-{
-  const guchar *expected_digest;
-
-  if (metadata_composefs == NULL)
-    return TRUE;
-
-  if (g_variant_n_children (metadata_composefs) != OSTREE_SHA256_DIGEST_LEN)
-    return glnx_throw (error, "Expected composefs fs-verity in metadata has the wrong size");
-
-  expected_digest = g_variant_get_data (metadata_composefs);
-  if (memcmp (fsverity_digest, expected_digest, OSTREE_SHA256_DIGEST_LEN) != 0)
-    {
-      char actual_checksum[OSTREE_SHA256_STRING_LEN + 1];
-      char expected_checksum[OSTREE_SHA256_STRING_LEN + 1];
-
-      ostree_checksum_inplace_from_bytes (fsverity_digest, actual_checksum);
-      ostree_checksum_inplace_from_bytes (expected_digest, expected_checksum);
-
-      return glnx_throw (error,
-                         "Generated composefs image digest (%s) doesn't match expected digest (%s)",
-                         actual_checksum, expected_checksum);
-    }
-
-  return TRUE;
-}
-
-#endif
-
 /* Look up @revision in the repository, and check it out in
  * /ostree/deploy/OS/deploy/${treecsum}.${deployserial}.
  * A dfd for the result is returned in @out_deployment_dfd.
@@ -696,54 +665,8 @@ checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeploy
     composefs_enabled = repo->composefs_wanted;
   if (composefs_enabled == OT_TRISTATE_YES)
     {
-      g_autofree guchar *fsverity_digest = NULL;
+      if (!ostree_repo_checkout_composefs (repo, NULL, ret_deployment_dfd, OSTREE_COMPOSEFS_NAME,
-      g_auto (GLnxTmpfile) tmpf = {
+                                           csum, cancellable, error))
-        0,
-      };
-      g_autoptr (GVariant) commit_variant = NULL;
-
-      if (!ostree_repo_load_commit (repo, revision, &commit_variant, NULL, error))
-        return FALSE;
-
-      g_autoptr (GVariant) metadata = g_variant_get_child_value (commit_variant, 0);
-      g_autoptr (GVariant) metadata_composefs = g_variant_lookup_value (
-          metadata, OSTREE_COMPOSEFS_DIGEST_KEY_V0, G_VARIANT_TYPE_BYTESTRING);
-
-      /* Create a composefs image and put in deploy dir */
-      g_autoptr (OstreeComposefsTarget) target = ostree_composefs_target_new ();
-
-      g_autoptr (GFile) commit_root = NULL;
-      if (!ostree_repo_read_commit (repo, csum, &commit_root, NULL, cancellable, error))
-        return FALSE;
-
-      if (!ostree_repo_checkout_composefs (repo, target, (OstreeRepoFile *)commit_root, cancellable,
-                                           error))
-        return FALSE;
-
-      g_autofree char *composefs_cfs_path
-          = g_strdup_printf ("%s/" OSTREE_COMPOSEFS_NAME, checkout_target_name);
-
-      g_debug ("writing %s", composefs_cfs_path);
-
-      if (!glnx_open_tmpfile_linkable_at (osdeploy_dfd, checkout_target_name, O_WRONLY | O_CLOEXEC,
-                                          &tmpf, error))
-        return FALSE;
-
-      if (!ostree_composefs_target_write (target, tmpf.fd, &fsverity_digest, cancellable, error))
-        return FALSE;
-
-      /* If the commit specified a composefs digest, verify it */
-      if (!compare_verity_digests (metadata_composefs, fsverity_digest, error))
-        return FALSE;
-
-      if (!glnx_fchmod (tmpf.fd, 0644, error))
-        return FALSE;
-
-      if (!_ostree_tmpf_fsverity (repo, &tmpf, NULL, error))
-        return FALSE;
-
-      if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, osdeploy_dfd, composefs_cfs_path,
-                                 error))
         return FALSE;
     }
   else

src/ostree/ot-builtin-checkout.c

@@ -28,6 +28,7 @@
 #include "ot-builtins.h"
 #include "otutil.h"
 
+static gboolean opt_composefs;
 static gboolean opt_user_mode;
 static gboolean opt_allow_noent;
 static gboolean opt_disable_cache;
@@ -107,6 +108,7 @@ static GOptionEntry options[] = {
     "PATH" },
   { "selinux-prefix", 0, 0, G_OPTION_ARG_STRING, &opt_selinux_prefix,
     "When setting SELinux labels, prefix all paths by PREFIX", "PREFIX" },
+  { "composefs", 0, 0, G_OPTION_ARG_NONE, &opt_composefs, "Only create a composefs blob", NULL },
   { NULL }
 };
 
@@ -136,10 +138,22 @@ process_one_checkout (OstreeRepo *repo, const char *resolved_commit, const char
    * `ostree_repo_checkout_at` until such time as we have a more
    * convenient infrastructure for testing C APIs with data.
    */
-  if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add || opt_force_copy
+  gboolean new_options_set = opt_disable_cache || opt_whiteouts || opt_require_hardlinks
-      || opt_force_copy_zerosized || opt_bareuseronly_dirs || opt_union_identical
+                             || opt_union_add || opt_force_copy || opt_force_copy_zerosized
-      || opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix
+                             || opt_bareuseronly_dirs || opt_union_identical || opt_skiplist_file
-      || opt_process_passthrough_whiteouts)
+                             || opt_selinux_policy || opt_selinux_prefix
+                             || opt_process_passthrough_whiteouts;
+
+  /* If we're doing composefs, then this is it */
+  if (opt_composefs)
+    {
+      if (new_options_set)
+        return glnx_throw (error, "Specified options are incompatible with --composefs");
+      return ostree_repo_checkout_composefs (repo, NULL, AT_FDCWD, destination, resolved_commit,
+                                             cancellable, error);
+    }
+
+  if (new_options_set)
     {
       OstreeRepoCheckoutAtOptions checkout_options = {
         0,

tests/test-composefs.sh

@@ -38,4 +38,14 @@ assert_streq "${orig_composefs_digest}" "${new_composefs_digest}"
 assert_streq "${new_composefs_digest}" "be956966c70970ea23b1a8043bca58cfb0d011d490a35a7817b36d04c0210954"
 tap_ok "composefs metadata"
 
+rm test2-co -rf
+$OSTREE checkout --composefs test-composefs test2-co.cfs
+digest=$(sha256sum < test2-co.cfs | cut -f 1 -d ' ')
+# This file should be reproducible bit for bit across environments; per above
+# we're operating on predictable data (fixed uid, gid, timestamps, xattrs, permissions).
+assert_streq "${digest}" "031fab2c7f390b752a820146dc89f6880e5739cba7490f64024e0c7d11aad7c9"
+# Verify it with composefs tooling
+composefs-info dump test2-co.cfs >/dev/null
+tap_ok "checkout composefs"
+
 tap_end