Support overlayfs whiteouts on checkout

Introduces an intermediate format for overlayfs storage, where
.wh-ostree. prefixed files will be converted into char 0:0
whiteout devices used by overlayfs to mark deletions across layers.

The CI scripts now uses a volume for the scratch directories
previously in /var/tmp otherwise we cannot create whiteout
devices into an overlayfs mounted filesystem.

Related-Issue: #2712
(cherry picked from commit e234b630f8)

.github/workflows/tests.yml

@@ -128,8 +128,14 @@ jobs:
       # An empty string isn't valid, so a dummy --label option is always
       # added.
       options: --label ostree ${{ matrix.container-options }}
+      # make sure tests are performed on a non-overlayfs filesystem
+      volumes:
+        - tmp_dir:/test-tmp
+      env:
+        TEST_TMPDIR: /test-tmp
 
     steps:
+
       - name: Pre-checkout setup
         run: ${{ matrix.pre-checkout-setup }}
         if: ${{ matrix.pre-checkout-setup }}
@@ -143,7 +149,7 @@ jobs:
         run: ./ci/gh-install.sh ${{ matrix.extra-packages }}
 
       - name: Add non-root user
-        run: "useradd builder && chown -R -h builder: ."
+        run: "useradd builder && chown -R -h builder: . $TEST_TMPDIR"
 
       - name: Build and test
         run: runuser -u builder -- ./ci/gh-build.sh ${{ matrix.configure-options }}

Makefile-tests.am

@@ -107,6 +107,7 @@ _installed_or_uninstalled_test_scripts = \
 	tests/test-admin-deploy-nomerge.sh \
 	tests/test-admin-deploy-none.sh \
 	tests/test-admin-deploy-bootid-gc.sh \
+	tests/test-admin-deploy-whiteouts.sh \
 	tests/test-osupdate-dtb.sh \
 	tests/test-admin-instutil-set-kargs.sh \
 	tests/test-admin-upgrade-not-backwards.sh \

bash/ostree

@@ -249,6 +249,7 @@ _ostree_checkout() {
         --union-identical
         --user-mode -U
         --whiteouts
+        --process-passthrough-whiteouts
     "
 
     local options_with_args="

man/ostree-checkout.xml

@@ -114,6 +114,17 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--process-passthrough-whiteouts</option></term>
+
+                <listitem><para>
+                   Enable overlayfs whiteout extraction into 0:0 character devices.
+                   Overlayfs whiteouts are encoded inside ostree as <literal>.ostree-wh.filename</literal>
+                   and extracted as 0:0 character devices. This is useful to carry
+                   container storage embedded into ostree.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--allow-noent</option></term>
 

src/libostree/ostree-repo-checkout.c

@@ -35,6 +35,8 @@
 #define WHITEOUT_PREFIX ".wh."
 #define OPAQUE_WHITEOUT_NAME ".wh..wh..opq"
 
+#define OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh."
+
 /* Per-checkout call state/caching */
 typedef struct {
   GString *path_buf; /* buffer for real path if filtering enabled */
@@ -582,6 +584,117 @@ checkout_file_hardlink (OstreeRepo                          *self,
   return TRUE;
 }
 
+static gboolean
+_checkout_overlayfs_whiteout_at_no_overwrite (OstreeRepoCheckoutAtOptions    *options,
+                                              int                             destination_dfd,
+                                              const char                     *destination_name,
+                                              GFileInfo                      *file_info,
+                                              GVariant                       *xattrs,
+                                              gboolean                       *found_exant_file,
+                                              GCancellable                   *cancellable,
+                                              GError                        **error)
+{
+  if (found_exant_file != NULL)
+    *found_exant_file = FALSE;
+  guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  if (mknodat(destination_dfd, destination_name, (file_mode & ~S_IFMT) | S_IFCHR, (dev_t)0) < 0)
+    {
+      if (errno == EEXIST && found_exant_file != NULL)
+        {
+          *found_exant_file = TRUE;
+          return TRUE;
+        }
+      return glnx_throw_errno_prefix (error, "Creating whiteout char device");
+    }
+  if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
+    {
+      if (xattrs != NULL &&
+          !glnx_dfd_name_set_all_xattrs(destination_dfd, destination_name, xattrs,
+                                          cancellable, error))
+          return glnx_throw_errno_prefix (error, "Setting xattrs for whiteout char device");
+
+      if (TEMP_FAILURE_RETRY(fchownat(destination_dfd, destination_name,
+                                      g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
+                                      g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
+                                      AT_SYMLINK_NOFOLLOW) < 0))
+          return glnx_throw_errno_prefix (error, "fchownat");
+      if (TEMP_FAILURE_RETRY (fchmodat (destination_dfd, destination_name, file_mode & ~S_IFMT, 0)) < 0)
+          return glnx_throw_errno_prefix (error, "fchmodat %s to 0%o", destination_name, file_mode & ~S_IFMT);
+    }
+
+  return TRUE;
+}
+
+static gboolean
+_checkout_overlayfs_whiteout_at (OstreeRepo                     *repo,
+                                 OstreeRepoCheckoutAtOptions    *options,
+                                 int                             destination_dfd,
+                                 const char                     *destination_name,
+                                 GFileInfo                      *file_info,
+                                 GVariant                       *xattrs,
+                                 GCancellable                   *cancellable,
+                                 GError                        **error)
+{
+  gboolean found_exant_file = FALSE;
+  if (!_checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
+                                                    file_info, xattrs,&found_exant_file,
+                                                    cancellable, error))
+    return FALSE;
+
+   if (!found_exant_file)
+    return TRUE;
+
+  guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+  guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+  guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+
+  struct stat dest_stbuf;
+
+  switch(options->overwrite_mode)
+    {
+      case OSTREE_REPO_CHECKOUT_OVERWRITE_NONE:
+        return FALSE;
+      case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES:
+        if (!ot_ensure_unlinked_at (destination_dfd, destination_name, error))
+          return FALSE;
+        return _checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
+                                                   file_info, xattrs, NULL, cancellable, error);
+      case OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES:
+        return TRUE;
+
+      case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL:
+        if (!glnx_fstatat(destination_dfd, destination_name, &dest_stbuf, AT_SYMLINK_NOFOLLOW,
+                          error))
+          return FALSE;
+        if (!(repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY))
+          {
+            g_autoptr(GVariant) fs_xattrs;
+            if (!glnx_dfd_name_get_all_xattrs (destination_dfd, destination_name,
+                                               &fs_xattrs, cancellable, error))
+              return FALSE;
+            if (!g_variant_equal(fs_xattrs, xattrs))
+              return glnx_throw(error, "existing destination file %s xattrs don't match",
+                                destination_name);
+          }
+        if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
+          {
+            if (gid != dest_stbuf.st_gid)
+              return glnx_throw(error, "existing destination file %s does not match gid %d",
+                                destination_name, gid);
+
+            if (uid != dest_stbuf.st_uid)
+              return glnx_throw(error, "existing destination file %s does not match uid %d",
+                                destination_name, gid);
+
+            if ((file_mode & ALLPERMS) != (dest_stbuf.st_mode & ALLPERMS))
+              return glnx_throw(error, "existing destination file %s does not match mode %o",
+                                destination_name, file_mode);
+          }
+        break;
+    }
+    return TRUE;
+}
+
 static gboolean
 checkout_one_file_at (OstreeRepo                        *repo,
                       OstreeRepoCheckoutAtOptions       *options,
@@ -603,7 +716,8 @@ checkout_one_file_at (OstreeRepo                        *repo,
 
   /* FIXME - avoid the GFileInfo here */
   g_autoptr(GFileInfo) source_info = NULL;
-  if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, NULL,
+  g_autoptr(GVariant) source_xattrs = NULL;
+  if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, &source_xattrs,
                               cancellable, error))
     return FALSE;
 
@@ -623,6 +737,7 @@ checkout_one_file_at (OstreeRepo                        *repo,
   const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0);
   const gboolean is_whiteout = (!is_symlink && options->process_whiteouts &&
                                 g_str_has_prefix (destination_name, WHITEOUT_PREFIX));
+  const gboolean is_overlayfs_whiteout = (!is_symlink && g_str_has_prefix (destination_name, OVERLAYFS_WHITEOUT_PREFIX));
   const gboolean is_reg_zerosized = (!is_symlink && g_file_info_get_size (source_info) == 0);
   const gboolean override_user_unreadable = (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER && is_unreadable);
 
@@ -643,6 +758,18 @@ checkout_one_file_at (OstreeRepo                        *repo,
 
       need_copy = FALSE;
     }
+  else if (is_overlayfs_whiteout && options->process_passthrough_whiteouts)
+    {
+      const char *name = destination_name + (sizeof (OVERLAYFS_WHITEOUT_PREFIX) - 1);
+
+      if (!name[0])
+        return glnx_throw (error, "Invalid empty overlayfs whiteout '%s'", name);
+
+      g_assert (name[0] != '/'); /* Sanity */
+
+      return _checkout_overlayfs_whiteout_at(repo, options, destination_dfd, name,
+                                             source_info, source_xattrs, cancellable, error);
+    }
   else if (is_reg_zerosized || override_user_unreadable)
     {
       /* In https://github.com/ostreedev/ostree/commit/673cacd633f9d6b653cdea530657d3e780a41bbd we

src/libostree/ostree-repo.h

@@ -989,8 +989,9 @@ typedef struct {
   gboolean force_copy; /* Since: 2017.6 */
   gboolean bareuseronly_dirs; /* Since: 2017.7 */
   gboolean force_copy_zerosized; /* Since: 2018.9 */
-  gboolean unused_bools[4];
+  gboolean process_passthrough_whiteouts;
-  /* 4 byte hole on 64 bit */
+  gboolean unused_bools[3];
+  /* 3 byte hole on 64 bit */
 
   const char *subpath;
 

src/libostree/ostree-sysroot-deploy.c

@@ -641,7 +641,7 @@ checkout_deployment_tree (OstreeSysroot     *sysroot,
     return FALSE;
 
   /* Generate hardlink farm, then opendir it */
-  OstreeRepoCheckoutAtOptions checkout_opts = { 0, };
+  OstreeRepoCheckoutAtOptions checkout_opts = { .process_passthrough_whiteouts = TRUE };
   if (!ostree_repo_checkout_at (repo, &checkout_opts, osdeploy_dfd,
                                 checkout_target_name, csum,
                                 cancellable, error))

src/ostree/ot-builtin-checkout.c

@@ -37,6 +37,7 @@ static gboolean opt_union;
 static gboolean opt_union_add;
 static gboolean opt_union_identical;
 static gboolean opt_whiteouts;
+static gboolean opt_process_passthrough_whiteouts;
 static gboolean opt_from_stdin;
 static char *opt_from_file;
 static gboolean opt_disable_fsync;
@@ -77,6 +78,7 @@ static GOptionEntry options[] = {
   { "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL },
   { "union-identical", 0, 0, G_OPTION_ARG_NONE, &opt_union_identical, "When layering checkouts, error out if a file would be replaced with a different version, but add new files and directories", NULL },
   { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL },
+  { "process-passthrough-whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_process_passthrough_whiteouts, "Enable overlayfs whiteout extraction into char 0:0 devices", NULL },
   { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL },
   { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL },
   { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" },
@@ -129,7 +131,8 @@ process_one_checkout (OstreeRepo           *repo,
   if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks ||
       opt_union_add || opt_force_copy || opt_force_copy_zerosized ||
       opt_bareuseronly_dirs || opt_union_identical ||
-      opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix)
+      opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix ||
+      opt_process_passthrough_whiteouts)
     {
       OstreeRepoCheckoutAtOptions checkout_options = { 0, };
 
@@ -162,6 +165,8 @@ process_one_checkout (OstreeRepo           *repo,
         }
       if (opt_whiteouts)
         checkout_options.process_whiteouts = TRUE;
+      if (opt_process_passthrough_whiteouts)
+        checkout_options.process_passthrough_whiteouts = TRUE;
       if (subpath)
         checkout_options.subpath = subpath;
 

tests/archive-test.sh

@@ -71,6 +71,11 @@ mkdir -p test-overlays
 date > test-overlays/overlaid-file
 $OSTREE commit ${COMMIT_ARGS} -b test-base --base test2 --owner-uid 42 --owner-gid 42 test-overlays/
 $OSTREE ls -R test-base > ls.txt
-assert_streq "$(wc -l < ls.txt)" 14
+if can_create_whiteout_devices; then
+    assert_streq "$(wc -l < ls.txt)" 17
+else
+    assert_streq "$(wc -l < ls.txt)" 14
+fi
+
 assert_streq "$(grep '42.*42' ls.txt | wc -l)" 2
 echo "ok commit overlay base"

tests/basic-test.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..$((87 + ${extra_basic_tests:-0}))"
+echo "1..$((89 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -1187,3 +1187,30 @@ if test "$(id -u)" != "0"; then
 else
     echo "ok # SKIP not run when root"
 fi
+
+if ! skip_one_without_whiteouts_devices; then
+    cd ${test_tmpdir}
+    rm checkout-test2 -rf
+    $OSTREE checkout test2 checkout-test2
+
+    assert_not_has_file checkout-test2/whiteouts/whiteout
+    assert_not_has_file checkout-test2/whiteouts/whiteout2
+    assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
+    assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
+
+    echo "ok checkout: no whiteout passthrough by default"
+fi
+
+if ! skip_one_without_whiteouts_devices; then
+    cd ${test_tmpdir}
+    rm checkout-test2 -rf
+    $OSTREE checkout --process-passthrough-whiteouts test2 checkout-test2
+
+    assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
+    assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
+
+    assert_is_whiteout_device checkout-test2/whiteouts/whiteout
+    assert_is_whiteout_device checkout-test2/whiteouts/whiteout2
+
+    echo "ok checkout: whiteout with overlayfs passthrough processing"
+fi

tests/kolainst/data-shared/libtest-core.sh

@@ -163,6 +163,13 @@ assert_file_has_mode () {
     fi
 }
 
+assert_is_whiteout_device () {
+    device_details="$(stat -c '%F %t:%T' $1)"
+    if [ "$device_details" != "character special file 0:0" ]; then
+        fatal "File '$1' is not a whiteout character device 0:0"
+    fi
+}
+
 assert_symlink_has_content () {
     if ! test -L "$1"; then
         fatal "File '$1' is not a symbolic link"

tests/libtest.sh

@@ -148,6 +148,20 @@ if ! have_selinux_relabel; then
 fi
 echo done
 
+# whiteout char 0:0 devices can be created as regular users, but
+# cannot be created inside containers mounted via overlayfs
+can_create_whiteout_devices() {
+    mknod -m 000 ${test_tmpdir}/.test-whiteout c 0 0 || return 1
+    rm -f ${test_tmpdir}/.test-whiteout
+    return 0
+}
+
+echo -n checking for overlayfs whiteouts...
+if ! can_create_whiteout_devices; then
+    export OSTREE_NO_WHITEOUTS=1
+fi
+echo done
+
 if test -n "${OT_TESTS_DEBUG:-}"; then
     set -x
 fi
@@ -245,6 +259,15 @@ setup_test_repository () {
     ln -s nonexistent baz/alink
     mkdir baz/another/
     echo x > baz/another/y
+
+    # if we are running inside a container we cannot test
+    # the overlayfs whiteout marker passthrough
+    if ! test -n "${OSTREE_NO_WHITEOUTS:-}"; then
+        mkdir whiteouts
+        touch whiteouts/.ostree-wh.whiteout
+        touch whiteouts/.ostree-wh.whiteout2
+        chmod 755 whiteouts/.ostree-wh.whiteout2
+    fi
     umask "${oldumask}"
 
     cd ${test_tmpdir}/files
@@ -406,7 +429,7 @@ setup_os_repository () {
     mkdir osdata
     cd osdata
     kver=3.6.0
-    mkdir -p usr/bin ${bootdir} usr/lib/modules/${kver} usr/share usr/etc
+    mkdir -p usr/bin ${bootdir} usr/lib/modules/${kver} usr/share usr/etc usr/container/layers/abcd
     kernel_path=${bootdir}/vmlinuz
     initramfs_path=${bootdir}/initramfs.img
     # the HMAC file is only in /usr/lib/modules
@@ -449,6 +472,17 @@ EOF
     mkdir -p usr/etc/testdirectory
     echo "a default daemon file" > usr/etc/testdirectory/test
 
+    # if we are running inside a container we cannot test
+    # the overlayfs whiteout marker passthrough
+    if ! test -n "${OSTREE_NO_WHITEOUTS:-}"; then
+        # overlayfs whiteout passhthrough marker files
+        touch usr/container/layers/abcd/.ostree-wh.whiteout
+        chmod 400 usr/container/layers/abcd/.ostree-wh.whiteout
+
+        touch usr/container/layers/abcd/.ostree-wh.whiteout2
+        chmod 777 usr/container/layers/abcd/.ostree-wh.whiteout2
+    fi
+
     ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit ${bootable_flag} --add-metadata-string version=1.0.9 -b testos/buildmain/x86_64-runtime -s "Build"
 
     # Ensure these commits have distinct second timestamps
@@ -588,6 +622,22 @@ skip_without_user_xattrs () {
     fi
 }
 
+# Usage: if ! skip_one_without_whiteouts_devices; then ... more tests ...; fi
+skip_one_without_whiteouts_devices() {
+    if ! can_create_whiteout_devices; then
+        echo "ok # SKIP - this test requires whiteout device support (test outside containers)"
+        return 0
+    else
+        return 1
+    fi
+}
+
+skip_without_whiteouts_devices () {
+    if ! can_create_whiteout_devices; then
+        skip "this test requires whiteout device support (test outside containers)"
+    fi
+}
+
 _have_systemd_and_libmount=''
 have_systemd_and_libmount() {
     if test "${_have_systemd_and_libmount}" = ''; then

tests/test-admin-deploy-whiteouts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright (C) 2022 Red Hat, Inc.
+#
+# SPDX-License-Identifier: LGPL-2.0+
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <https://www.gnu.org/licenses/>.
+
+set -euox pipefail
+
+. $(dirname $0)/libtest.sh
+
+skip_without_whiteouts_devices
+
+# Exports OSTREE_SYSROOT so --sysroot not needed.
+setup_os_repository "archive" "syslinux"
+${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmain/x86_64-runtime
+
+echo "1..3"
+${CMD_PREFIX} ostree admin deploy --os=testos --karg=root=LABEL=foo --karg=testkarg=1 testos:testos/buildmain/x86_64-runtime
+origdeployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir)
+
+assert_is_whiteout_device "${origdeployment}"/usr/container/layers/abcd/whiteout
+echo "ok whiteout deployment"
+
+assert_not_has_file  "${origdeployment}"/usr/container/layers/abcd/.ostree-wh.whiteout
+echo "ok .ostree-wh.whiteout not created"
+
+assert_file_has_mode "${origdeployment}"/usr/container/layers/abcd/whiteout 400
+assert_file_has_mode "${origdeployment}"/usr/container/layers/abcd/whiteout2 777
+echo "ok whiteout permissions are preserved"