Support Docker-style whiteouts

This is to enable importing Docker layers as ostree commits, then
checking them out in a union.

The prototype work for this is in:
https://github.com/cgwalters/dlayer-ostree

Though it will likely ultimately end up in:
https://github.com/projectatomic/atomic

src/libostree/ostree-repo-checkout.c

@@ -32,6 +32,8 @@
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
 
+#define WHITEOUT_PREFIX ".wh."
+
 static gboolean
 checkout_object_for_uncompressed_cache (OstreeRepo      *self,
                                         const char      *loose_path,
@@ -396,20 +398,46 @@ checkout_one_file_at (OstreeRepo                        *repo,
   const char *checksum;
   gboolean is_symlink;
   gboolean can_cache;
-  gboolean did_hardlink = FALSE;
+  gboolean need_copy = TRUE;
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
   g_autoptr(GInputStream) input = NULL;
   g_autoptr(GVariant) xattrs = NULL;
+  gboolean is_whiteout;
 
   is_symlink = g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK;
 
   checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source);
 
-  /* Try to do a hardlink first, if it's a regular file.  This also
-   * traverses all parent repos.
+  is_whiteout = !is_symlink && options->process_whiteouts &&
+    g_str_has_prefix (destination_name, WHITEOUT_PREFIX);
+
+  /* First, see if it's a Docker whiteout,
+   * https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go
    */
-  if (!is_symlink)
+  if (is_whiteout)
     {
+      const char *name = destination_name + (sizeof (WHITEOUT_PREFIX) - 1);
+
+      if (!name[0])
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Invalid empty whiteout '%s'", name);
+          goto out;
+        }
+
+      g_assert (name[0] != '/'); /* Sanity */
+
+      if (!glnx_shutil_rm_rf_at (destination_dfd, name, cancellable, error))
+        goto out;
+
+      need_copy = FALSE;
+    }
+  else if (!is_symlink)
+    {
+      gboolean did_hardlink = FALSE;
+      /* Try to do a hardlink first, if it's a regular file.  This also
+       * traverses all parent repos.
+       */
       OstreeRepo *current_repo = repo;
 
       while (current_repo)
@@ -462,6 +490,8 @@ checkout_one_file_at (OstreeRepo                        *repo,
             }
           current_repo = current_repo->parent_repo;
         }
+
+      need_copy = !did_hardlink;
     }
 
   can_cache = (options->enable_uncompressed_cache
@@ -471,11 +501,14 @@ checkout_one_file_at (OstreeRepo                        *repo,
    * it now, stick it in the cache, and then hardlink to that.
    */
   if (can_cache
+      && !is_whiteout
       && !is_symlink
-      && !did_hardlink
+      && need_copy
       && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
       && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER)
     {
+      gboolean did_hardlink;
+      
       if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL,
                                   cancellable, error))
         goto out;
@@ -526,10 +559,12 @@ checkout_one_file_at (OstreeRepo                        *repo,
           g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name);
           goto out;
         }
+
+      need_copy = !did_hardlink;
     }
 
   /* Fall back to copy if we couldn't hardlink */
-  if (!did_hardlink)
+  if (need_copy)
     {
       if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs,
                                   cancellable, error))

src/libostree/ostree-repo.h

@@ -557,7 +557,8 @@ typedef struct {
   
   guint enable_uncompressed_cache : 1;
   guint disable_fsync : 1;
-  guint reserved : 30;
+  guint process_whiteouts : 1;
+  guint reserved : 29;
 
   const char *subpath;
 

src/ostree/ot-builtin-checkout.c

@@ -36,6 +36,7 @@ static gboolean opt_allow_noent;
 static gboolean opt_disable_cache;
 static char *opt_subpath;
 static gboolean opt_union;
+static gboolean opt_whiteouts;
 static gboolean opt_from_stdin;
 static char *opt_from_file;
 static gboolean opt_disable_fsync;
@@ -61,6 +62,7 @@ static GOptionEntry options[] = {
   { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL },
   { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" },
   { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL },
+  { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL },
   { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL },
   { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL },
   { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" },
@@ -83,7 +85,7 @@ process_one_checkout (OstreeRepo           *repo,
    * `ostree_repo_checkout_tree_at` until such time as we have a more
    * convenient infrastructure for testing C APIs with data.
    */
-  if (opt_disable_cache)
+  if (opt_disable_cache || opt_whiteouts)
     {
       OstreeRepoCheckoutOptions options = { 0, };
       
@@ -91,10 +93,11 @@ process_one_checkout (OstreeRepo           *repo,
         options.mode = OSTREE_REPO_CHECKOUT_MODE_USER;
       if (opt_union)
         options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES;
+      if (opt_whiteouts)
+        options.process_whiteouts = TRUE;
       if (subpath)
         options.subpath = subpath;
 
-
       if (!ostree_repo_checkout_tree_at (repo, &options,
                                          AT_FDCWD, destination,
                                          resolved_commit,

tests/basic-test.sh

@@ -387,6 +387,42 @@ assert_file_has_content test2-checkout/baz/cow moo
 assert_has_dir repo2/uncompressed-objects-cache
 echo "ok disable cache checkout"
 
+# Whiteouts
+cd ${test_tmpdir}
+mkdir -p overlay/baz/
+touch overlay/baz/.wh.cow
+touch overlay/.wh.deeper
+touch overlay/anewfile
+mkdir overlay/anewdir/
+touch overlay/anewdir/blah
+$OSTREE --repo=repo commit -b overlay -s 'overlay' --tree=dir=overlay
+rm overlay -rf
+
+for branch in test2 overlay; do
+    $OSTREE --repo=repo checkout --union --whiteouts ${branch} overlay-co
+done
+for f in .wh.deeper baz/cow baz/.wh.cow; do
+    assert_not_has_file overlay-co/${f}
+done
+assert_not_has_dir overlay-co/deeper
+assert_has_file overlay-co/anewdir/blah
+assert_has_file overlay-co/anewfile
+
+echo "ok whiteouts enabled"
+
+# Now double check whiteouts are not processed without --whiteouts 
+rm overlay-co -rf
+for branch in test2 overlay; do
+    $OSTREE --repo=repo checkout --union ${branch} overlay-co
+done
+for f in .wh.deeper baz/cow baz/.wh.cow; do
+    assert_has_file overlay-co/${f}
+done
+assert_not_has_dir overlay-co/deeper
+assert_has_file overlay-co/anewdir/blah
+assert_has_file overlay-co/anewfile
+echo "ok whiteouts disabled"
+
 cd ${test_tmpdir}
 rm -rf test2-checkout
 mkdir -p test2-checkout