Rework treecompose kernel processing

Prep for changing `boot_location: new` to use `/usr/lib/ostree-boot`
and `/usr/lib/modules`.  Rework our kernel postprocessing
so that we unify the `boot_location` handling with initramfs generation.

Instead of doing the initramfs first in postprocessing, we do it nearly last,
after e.g. `etc` is renamed to `usr/etc`. This has some consequences, such as
the fact that `run_bwrap_mutably()` is now called in both situations. In
general, our handling of `etc` is inconsistent, although understandably so.

As part of this, I finally got around to implementing the bit from
https://github.com/systemd/systemd/pull/4174 however suboptimal it is; need the
unified core so we can cleanly ignore the posttrans like we do others.  We
intentionally keep the file around in the generated tree so that installing a
kernel RPM per client doesn't try to do any of this either.

This all gets folded together so that the logic for handling the bootloader gets
simpler - in the Fedora case, we now know to find kernels in `/usr/lib/modules`
and can ignore `/boot`.

Closes: #959
Approved by: jlebon

docs/manual/treefile.md

@@ -32,7 +32,7 @@ It supports the following parameters:
     possible values:
     * "both": the default, kernel data goes in /boot and /usr/lib/ostree-boot
     * "legacy": Now an alias for "both"; historically meant just "boot"
-    * "new": kernel data goes in /usr/lib/ostree-boot
+    * "new": kernel data goes in /usr/lib/ostree-boot and /usr/lib/modules
 
  * `etc-group-members`: Array of strings, optional: Unix groups in this
    list will be stored in `/etc/group` instead of `/usr/lib/group`.  Use

src/app/rpmostree-compose-builtin-tree.c

@@ -477,6 +477,21 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
         return FALSE;
     }
 
+  /* Before we install packages, drop a file to suppress the kernel.rpm dracut run.
+   * <https://github.com/systemd/systemd/pull/4174> */
+  const char *kernel_installd_path = "usr/lib/kernel/install.d";
+  if (!glnx_shutil_mkdir_p_at (rootfs_dfd, kernel_installd_path, 0755, cancellable, error))
+    return FALSE;
+  const char skip_kernel_install_data[] = "#!/usr/bin/sh\nexit 77\n";
+  const char *kernel_skip_path = glnx_strjoina (kernel_installd_path, "/00-rpmostree-skip.install");
+  if (!glnx_file_replace_contents_with_perms_at (rootfs_dfd, kernel_skip_path,
+                                                 (guint8*)skip_kernel_install_data,
+                                                 strlen (skip_kernel_install_data),
+                                                 0755, 0, 0,
+                                                 GLNX_FILE_REPLACE_NODATASYNC,
+                                                 cancellable, error))
+    return FALSE;
+
   /* Now actually run through librpm to install the packages.  Note this bit
    * will be replaced in the future with a unified core:
    * https://github.com/projectatomic/rpm-ostree/issues/729

src/daemon/rpmostree-sysroot-upgrader.c

@@ -962,7 +962,7 @@ perform_local_assembly (RpmOstreeSysrootUpgrader *self,
         return FALSE;
 
       if (!rpmostree_finalize_kernel (self->tmprootfs_dfd, bootdir, kver, kernel_path,
-                                      &initramfs_tmpf,
+                                      &initramfs_tmpf, RPMOSTREE_FINALIZE_KERNEL_AUTO,
                                       cancellable, error))
         return FALSE;
 

src/libpriv/rpmostree-kernel.c

@@ -38,6 +38,12 @@
 #include "rpmostree-bwrap.h"
 #include "rpmostree-util.h"
 
+static const char usrlib_ostreeboot[] = "usr/lib/ostree-boot";
+
+/* Keep this in sync with ostree/src/libostree/ostree-sysroot-deploy.c:get_kernel_from_tree().
+ * Note they are of necessity slightly different since rpm-ostree needs
+ * to support grabbing wherever the Fedora kernel RPM dropped files as well.
+ */
 static gboolean
 find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
                                       const char  *bootdir,
@@ -57,7 +63,7 @@ find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
   if (dfd < 0)
     {
       if (errno == ENOENT)
-        return TRUE;
+        return TRUE; /* Note early return */
       else
         return glnx_throw_errno_prefix (error, "opendir(%s)", bootdir);
     }
@@ -80,25 +86,16 @@ find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
       name = dent->d_name;
 
       /* Current Fedora 23 kernel.spec installs as just vmlinuz */
-      if (strcmp (name, "vmlinuz") == 0 || g_str_has_prefix (name, "vmlinuz-"))
+      if (g_str_equal (name, "vmlinuz") || g_str_has_prefix (name, "vmlinuz-"))
         {
           if (ret_kernel)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Multiple vmlinuz- in %s",
-                           bootdir);
-              return FALSE;
-            }
+            return glnx_throw (error, "Multiple vmlinuz- in %s", bootdir);
           ret_kernel = g_strconcat (bootdir, "/", name, NULL);
         }
-      else if (g_str_has_prefix (name, "initramfs-"))
+      else if (g_str_equal (name, "initramfs.img") || g_str_has_prefix (name, "initramfs-"))
         {
           if (ret_initramfs)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Multiple initramfs- in %s", bootdir);
-              return FALSE;
-            }
+            return glnx_throw (error, "Multiple initramfs- in %s", bootdir);
           ret_initramfs = g_strconcat (bootdir, "/", name, NULL);
         }
     }
@@ -153,8 +150,8 @@ find_ensure_one_subdirectory (int            rootfs_dfd,
 /* Given a root filesystem, return a GVariant of format (sssms):
  *  - kver: uname -r equivalent
  *  - bootdir: Path to the boot directory
- *  - kernel_path: Relative path to kernel
- *  - initramfs_path: Relative path to initramfs (may be NULL if no initramfs)
+ *  - kernel_path: Relative (to rootfs) path to kernel
+ *  - initramfs_path: Relative (to rootfs) path to initramfs (may be NULL if no initramfs)
  */
 GVariant *
 rpmostree_find_kernel (int rootfs_dfd,
@@ -175,7 +172,7 @@ rpmostree_find_kernel (int rootfs_dfd,
   /* First, look for the kernel in the canonical ostree directory */
   g_autofree char* kernel_path = NULL;
   g_autofree char* initramfs_path = NULL;
-  g_autofree char *bootdir = g_strdup ("usr/lib/ostree-boot");
+  g_autofree char *bootdir = g_strdup (usrlib_ostreeboot);
   if (!find_kernel_and_initramfs_in_bootdir (rootfs_dfd, bootdir,
                                              &kernel_path, &initramfs_path,
                                              cancellable, error))
@@ -211,8 +208,64 @@ rpmostree_find_kernel (int rootfs_dfd,
   return g_variant_ref_sink (g_variant_new ("(sssms)", kver, bootdir, kernel_path, initramfs_path));
 }
 
-/* Given a kernel path and a temporary initramfs, compute their checksum and put
- * them in their final locations.
+/* Given a @rootfs_dfd and path to kernel/initramfs that live in
+ * usr/lib/modules/$kver, possibly update @bootdir to use them. @bootdir should
+ * be one of either /usr/lib/ostree-boot or /boot. If @only_if_found is set, we
+ * do the copy only if we find a kernel; this way we avoid e.g. touching /boot
+ * if it isn't being used.
+ */
+static gboolean
+copy_kernel_into (int rootfs_dfd,
+                  const char *kver,
+                  const char *boot_checksum_str,
+                  const char *kernel_modules_path,
+                  const char *initramfs_modules_path,
+                  gboolean only_if_found,
+                  const char *bootdir,
+                  GCancellable *cancellable,
+                  GError **error)
+{
+  g_autofree char *legacy_kernel_path = NULL;
+  g_autofree char* legacy_initramfs_path = NULL;
+  if (!find_kernel_and_initramfs_in_bootdir (rootfs_dfd, bootdir,
+                                             &legacy_kernel_path, &legacy_initramfs_path,
+                                             cancellable, error))
+    return FALSE;
+
+  /* No kernel found? Skip to the next if we're in "auto"
+   * mode i.e. only update if found.
+   */
+  if (!legacy_kernel_path && only_if_found)
+    return TRUE;
+
+  /* Update kernel */
+  if (legacy_kernel_path)
+    {
+      if (!glnx_unlinkat (rootfs_dfd, legacy_kernel_path, 0, error))
+        return FALSE;
+      g_free (legacy_kernel_path);
+    }
+  legacy_kernel_path = g_strconcat (bootdir, "/", "vmlinuz-", kver, "-", boot_checksum_str, NULL);
+  if (linkat (rootfs_dfd, kernel_modules_path, rootfs_dfd, legacy_kernel_path, 0) < 0)
+    return glnx_throw_errno_prefix (error, "linkat(%s)", legacy_kernel_path);
+
+  /* Update initramfs */
+  if (legacy_initramfs_path)
+    {
+      if (!glnx_unlinkat (rootfs_dfd, legacy_initramfs_path, 0, error))
+        return FALSE;
+      g_free (legacy_initramfs_path);
+    }
+  legacy_initramfs_path = g_strconcat (bootdir, "/", "initramfs-", kver, ".img-", boot_checksum_str, NULL);
+  if (linkat (rootfs_dfd, initramfs_modules_path, rootfs_dfd, legacy_initramfs_path, 0) < 0)
+    return glnx_throw_errno_prefix (error, "linkat(%s)", legacy_initramfs_path);
+
+  return TRUE;
+}
+
+/* Given a kernel path and a temporary initramfs, place them in their final
+ * location. We handle /usr/lib/modules as well as the /usr/lib/ostree-boot and
+ * /boot paths where we need to pre-compute their checksum.
  */
 gboolean
 rpmostree_finalize_kernel (int rootfs_dfd,
@@ -220,43 +273,81 @@ rpmostree_finalize_kernel (int rootfs_dfd,
                            const char *kver,
                            const char *kernel_path,
                            GLnxTmpfile *initramfs_tmpf,
+                           RpmOstreeFinalizeKernelDestination dest,
                            GCancellable *cancellable,
                            GError **error)
 {
-  g_autoptr(GChecksum) boot_checksum = NULL;
-  g_autofree char *kernel_final_path = NULL;
-  g_autofree char *initramfs_final_path = NULL;
-  const char *boot_checksum_str = NULL;
+  const char slash_bootdir[] = "boot";
+  g_autofree char *modules_bootdir = g_strconcat ("usr/lib/modules/", kver, NULL);
 
-  /* Now, calculate the combined sha256sum of the two. We checksum the initramfs
-   * from the tmpfile fd (via mmap()) to avoid writing it to disk in another
-   * temporary location.
+  /* Calculate the sha256sum of the kernel+initramfs (called the "boot
+   * checksum"). We checksum the initramfs from the tmpfile fd (via mmap()) to
+   * avoid writing it to disk in another temporary location.
    */
-  boot_checksum = g_checksum_new (G_CHECKSUM_SHA256);
+  g_autoptr(GChecksum) boot_checksum = g_checksum_new (G_CHECKSUM_SHA256);
   if (!_rpmostree_util_update_checksum_from_file (boot_checksum, rootfs_dfd, kernel_path,
                                                   cancellable, error))
     return FALSE;
-
   { g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (initramfs_tmpf->fd, FALSE, error);
     if (!mfile)
       return FALSE;
     g_checksum_update (boot_checksum, (guint8*)g_mapped_file_get_contents (mfile),
                        g_mapped_file_get_length (mfile));
   }
-  boot_checksum_str = g_checksum_get_string (boot_checksum);
+  const char *boot_checksum_str = g_checksum_get_string (boot_checksum);
 
-  kernel_final_path = g_strconcat (bootdir, "/", "vmlinuz-", kver, "-", boot_checksum_str, NULL);
-  initramfs_final_path = g_strconcat (bootdir, "/", "initramfs-", kver, ".img-", boot_checksum_str, NULL);
+  g_autofree char *kernel_modules_path = g_strconcat (modules_bootdir, "/vmlinuz", NULL);;
+  /* It's possible the bootdir is already the modules directory; in that case,
+   * we don't need to rename.
+   */
+  if (!g_str_equal (kernel_path, kernel_modules_path))
+    {
+      g_assert_cmpstr (bootdir, !=, modules_bootdir);
+      /* Ensure that the /usr/lib/modules kernel is the same as the source.
+       * Right now we don't support overriding the kernel, but to be
+       * conservative let's relink (unlink/link). We don't just rename() because
+       * for _AUTO mode we still want to find the kernel in the old path
+       * (probably /usr/lib/ostree-boot) and update as appropriate.
+       */
+      if (unlinkat (rootfs_dfd, kernel_modules_path, 0) < 0)
+        {
+          if (errno != ENOENT)
+            return glnx_throw_errno_prefix (error, "unlinkat(%s)", kernel_modules_path);
+        }
+      if (linkat (rootfs_dfd, kernel_path, rootfs_dfd, kernel_modules_path, 0) < 0)
+        return glnx_throw_errno_prefix (error, "linkat(%s)", kernel_modules_path);
+    }
 
-  /* Put the kernel in the final location */
-  if (!glnx_renameat (rootfs_dfd, kernel_path, rootfs_dfd, kernel_final_path, error))
-    return FALSE;
-  /* Link the initramfs directly to its final destination */
+  /* Replace the initramfs */
+  g_autofree char *initramfs_modules_path = g_strconcat (modules_bootdir, "/initramfs.img", NULL);
+  if (unlinkat (rootfs_dfd, initramfs_modules_path, 0) < 0)
+    {
+      if (errno != ENOENT)
+        return glnx_throw_errno_prefix (error, "unlinkat(%s)", initramfs_modules_path);
+    }
   if (!glnx_link_tmpfile_at (initramfs_tmpf, GLNX_LINK_TMPFILE_NOREPLACE,
-                             rootfs_dfd, initramfs_final_path,
+                             rootfs_dfd, initramfs_modules_path,
                              error))
     return FALSE;
 
+  /* Update /usr/lib/ostree-boot and /boot (if desired) */
+  const gboolean only_if_found = (dest == RPMOSTREE_FINALIZE_KERNEL_AUTO);
+  if (only_if_found || dest >= RPMOSTREE_FINALIZE_KERNEL_USRLIB_OSTREEBOOT)
+    {
+      if (!copy_kernel_into (rootfs_dfd, kver, boot_checksum_str,
+                             kernel_modules_path, initramfs_modules_path,
+                             only_if_found, usrlib_ostreeboot,
+                             cancellable, error))
+        return FALSE;
+    }
+  if (only_if_found || dest >= RPMOSTREE_FINALIZE_KERNEL_SLASH_BOOT)
+    {
+      if (!copy_kernel_into (rootfs_dfd, kver, boot_checksum_str,
+                             kernel_modules_path, initramfs_modules_path,
+                             only_if_found, slash_bootdir,
+                             cancellable, error))
+        return FALSE;
+    }
   return TRUE;
 }
 
@@ -341,12 +432,16 @@ rpmostree_run_dracut (int     rootfs_dfd,
                                       &tmpf, error))
     goto out;
 
+  /* If we're rebuilding, we use the *current* /etc so we pick up any modified
+   * config files.  Otherwise, we use the usr/etc defaults.
+   */
   if (rebuild_from_initramfs)
     bwrap = rpmostree_bwrap_new (rootfs_dfd, RPMOSTREE_BWRAP_IMMUTABLE, error,
                                  "--ro-bind", "/etc", "/etc",
                                  NULL);
   else
     bwrap = rpmostree_bwrap_new (rootfs_dfd, RPMOSTREE_BWRAP_IMMUTABLE, error,
+                                 "--ro-bind", "usr/etc", "/etc",
                                  NULL);
   if (!bwrap)
     return FALSE;

src/libpriv/rpmostree-kernel.h

@@ -22,6 +22,13 @@
 
 #include <ostree.h>
 
+typedef enum {
+  RPMOSTREE_FINALIZE_KERNEL_AUTO,
+  RPMOSTREE_FINALIZE_KERNEL_USRLIB_MODULES,
+  RPMOSTREE_FINALIZE_KERNEL_USRLIB_OSTREEBOOT,
+  RPMOSTREE_FINALIZE_KERNEL_SLASH_BOOT,
+} RpmOstreeFinalizeKernelDestination;
+
 GVariant *
 rpmostree_find_kernel (int rootfs_dfd,
                        GCancellable *cancellable,
@@ -33,6 +40,7 @@ rpmostree_finalize_kernel (int rootfs_dfd,
                            const char *kver,
                            const char *kernel_path,
                            GLnxTmpfile *initramfs_tmpf,
+                           RpmOstreeFinalizeKernelDestination dest,
                            GCancellable *cancellable,
                            GError **error);
 

src/libpriv/rpmostree-postprocess.c

@@ -59,12 +59,24 @@ run_bwrap_mutably (int           rootfs_fd,
                    char        **child_argv,
                    GError     **error)
 {
-  g_autoptr(RpmOstreeBwrap) bwrap = NULL;
+  struct stat stbuf;
+  const char *etc_bind;
 
-  bwrap = rpmostree_bwrap_new (rootfs_fd, RPMOSTREE_BWRAP_MUTATE_FREELY, error,
-                               "--bind", "var", "/var",
-                               "--bind", "etc", "/etc",
-                               NULL);
+  /* This gets called both by treecompose, where in the non-unified path we just
+   * have /etc, and in kernel postprocessing where we have usr/etc.
+   */
+  if (!glnx_fstatat_allow_noent (rootfs_fd, "etc", &stbuf, 0, error))
+    return FALSE;
+  if (errno == ENOENT)
+    etc_bind = "usr/etc";
+  else
+    etc_bind = "etc";
+
+  g_autoptr(RpmOstreeBwrap) bwrap =
+    rpmostree_bwrap_new (rootfs_fd, RPMOSTREE_BWRAP_MUTATE_FREELY, error,
+                         "--bind", "var", "/var",
+                         "--bind", etc_bind, "/etc",
+                         NULL);
   if (!bwrap)
     return FALSE;
 
@@ -142,60 +154,186 @@ init_rootfs (int            dfd,
   return TRUE;
 }
 
+/* Given a directory referenced by @src_dfd+@src_path,
+ * Create @dest_dfd+@dest_path as a directory, hardlinking
+ * all content - recursively.
+ */
 static gboolean
-do_kernel_prep (int            rootfs_dfd,
-                JsonObject    *treefile,
-                GCancellable  *cancellable,
-                GError       **error)
+hardlink_recurse (int                src_dfd,
+                  const char        *src_path,
+                  int                dest_dfd,
+                  const char        *dest_path,
+                  GCancellable      *cancellable,
+                  GError            **error)
 {
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  glnx_fd_close int dest_target_dfd = -1;
+
+  if (!glnx_dirfd_iterator_init_at (src_dfd, src_path, TRUE, &dfd_iter, error))
+    return FALSE;
+
+  if (!glnx_opendirat (dest_dfd, dest_path, TRUE, &dest_target_dfd, error))
+    return FALSE;
+
+  while (TRUE)
+    {
+      struct dirent *dent = NULL;
+      struct stat stbuf;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
+        return FALSE;
+      if (!dent)
+        break;
+
+      if (!glnx_fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+        return FALSE;
+
+      if (dent->d_type == DT_DIR)
+        {
+          mode_t perms = stbuf.st_mode & ~S_IFMT;
+
+          if (!glnx_ensure_dir (dest_target_dfd, dent->d_name, perms, error))
+            return FALSE;
+          if (fchmodat (dest_target_dfd, dent->d_name, perms, 0) < 0)
+            return glnx_throw_errno_prefix (error, "fchmodat");
+          if (!hardlink_recurse (dfd_iter.fd, dent->d_name,
+                                 dest_target_dfd, dent->d_name,
+                                 cancellable, error))
+            return FALSE;
+        }
+      else
+        {
+          if (linkat (dfd_iter.fd, dent->d_name,
+                      dest_target_dfd, dent->d_name, 0) < 0)
+            return glnx_throw_errno_prefix (error, "linkat");
+        }
+    }
+
+  return TRUE;
+}
+
+/* Handle the kernel/initramfs, which can be in at least 2 different places:
+ *  - /boot (CentOS, Fedora treecompose before we suppressed kernel.spec's %posttrans)
+ *  - /usr/lib/modules (Fedora treecompose without kernel.spec's %posttrans)
+ *
+ * We then need to handle the boot_location option, which can put that data in
+ * either both (/boot and /usr/lib/ostree-boot), or just the latter.
+ */
+static gboolean
+process_kernel_and_initramfs (int            rootfs_dfd,
+                              JsonObject    *treefile,
+                              GCancellable  *cancellable,
+                              GError       **error)
+{
+  /* The current systemd kernel-install will inject
+   * /boot/${machine_id}/${uname -r} which we don't use;
+   * to avoid confusion, we will delete it.  This relies
+   * on systemd itself having set up the machine id from its %post,
+   * so we need to read it.  We'll reset the machine ID after this.
+   */
+  { glnx_fd_close int fd = openat (rootfs_dfd, "usr/etc/machine-id", O_RDONLY | O_CLOEXEC);
+    if (fd < 0)
+      {
+        if (errno != ENOENT)
+          return glnx_throw_errno_prefix (error, "openat(usr/etc/machine-id)");
+      }
+    else
+      {
+        g_autofree char *old_machine_id = glnx_fd_readall_utf8 (fd, NULL, cancellable, error);
+        if (!old_machine_id)
+          return FALSE;
+        if (strlen (old_machine_id) != 33)
+          return glnx_throw (error, "invalid machine ID '%.33s'", old_machine_id);
+        /* Trim newline */
+        old_machine_id[32] = '\0';
+
+        const char *boot_machineid_dir = glnx_strjoina ("boot/", old_machine_id);
+        if (!glnx_shutil_rm_rf_at (rootfs_dfd, boot_machineid_dir, cancellable, error))
+          return FALSE;
+      }
+  }
+
+  /* We need to move non-kernel data (bootloader bits usually) into
+   * /usr/lib/ostree-boot; this will also take care of moving the kernel in legacy
+   * paths (CentOS, Fedora <= 24), etc.
+   */
+  if (!glnx_renameat (rootfs_dfd, "boot", rootfs_dfd, "usr/lib/ostree-boot", error))
+    return FALSE;
+
+  /* Find the kernel in the source root (at this point one of usr/lib/modules or
+   * usr/lib/ostree-boot)
+   */
   g_autoptr(GVariant) kernelstate = rpmostree_find_kernel (rootfs_dfd, cancellable, error);
   if (!kernelstate)
     return FALSE;
-
   const char* kernel_path;
   const char* initramfs_path;
   const char *kver;
   const char *bootdir;
+  /* Used to optionally hardlink result of our dracut run */
   g_variant_get (kernelstate, "(&s&s&sm&s)",
                  &kver, &bootdir,
                  &kernel_path, &initramfs_path);
 
+  /* We generate our own initramfs with custom arguments, so if the RPM install
+   * generated one (should only happen on CentOS now), delete it.
+   */
   if (initramfs_path)
     {
+      g_assert_cmpstr (bootdir, ==, "usr/lib/ostree-boot");
+      g_assert_cmpint (*initramfs_path, !=, '/');
       g_print ("Removing RPM-generated '%s'\n", initramfs_path);
       if (!glnx_shutil_rm_rf_at (rootfs_dfd, initramfs_path, cancellable, error))
         return FALSE;
+      initramfs_path = NULL;
     }
 
-  /* OSTree needs to own this */
-  if (!glnx_shutil_rm_rf_at (rootfs_dfd, "boot/loader", cancellable, error))
-    return FALSE;
-
+  /* Ensure depmod (kernel modules index) is up to date; because on Fedora we
+   * suppress the kernel %posttrans we need to take care of this.
+   */
   {
     char *child_argv[] = { "depmod", (char*)kver, NULL };
     if (!run_bwrap_mutably (rootfs_dfd, "depmod", child_argv, error))
       return FALSE;
   }
 
-  /* Ensure the /etc/machine-id file is present and empty. Apparently systemd
-     doesn't work when the file is missing (as of systemd-219-9.fc22) but it is
-     correctly populated if the file is there.  */
+  RpmOstreePostprocessBootLocation boot_location =
+    RPMOSTREE_POSTPROCESS_BOOT_LOCATION_BOTH;
+  const char *boot_location_str = NULL;
+  if (!_rpmostree_jsonutil_object_get_optional_string_member (treefile,
+                                                              "boot_location",
+                                                              &boot_location_str, error))
+    return FALSE;
+  if (boot_location_str != NULL)
+    {
+      /* Make "legacy" an alias for "both" */
+      if (strcmp (boot_location_str, "both") == 0 ||
+          strcmp (boot_location_str, "legacy") == 0)
+        ;
+      else if (strcmp (boot_location_str, "new") == 0)
+        boot_location = RPMOSTREE_POSTPROCESS_BOOT_LOCATION_NEW;
+      else
+        return glnx_throw (error, "Invalid boot location '%s'", boot_location_str);
+    }
+
+  /* Ensure the /etc/machine-id file is present and empty; it is read by
+   * dracut. Apparently systemd doesn't work when the file is missing (as of
+   * systemd-219-9.fc22) but it is correctly populated if the file is there.
+   */
   g_print ("Creating empty machine-id\n");
-  if (!glnx_file_replace_contents_at (rootfs_dfd, "etc/machine-id", (guint8*)"", 0,
+  if (!glnx_file_replace_contents_at (rootfs_dfd, "usr/etc/machine-id", (guint8*)"", 0,
                                       GLNX_FILE_REPLACE_NODATASYNC,
                                       cancellable, error))
     return FALSE;
 
+  /* Run dracut with our chosen arguments (commonly at least --no-hostonly) */
   g_autoptr(GPtrArray) dracut_argv = g_ptr_array_new ();
   if (json_object_has_member (treefile, "initramfs-args"))
     {
-      guint i, len;
-      JsonArray *initramfs_args;
+      JsonArray *initramfs_args = json_object_get_array_member (treefile, "initramfs-args");
+      guint len = json_array_get_length (initramfs_args);
 
-      initramfs_args = json_object_get_array_member (treefile, "initramfs-args");
-      len = json_array_get_length (initramfs_args);
-
-      for (i = 0; i < len; i++)
+      for (guint i = 0; i < len; i++)
         {
           const char *arg = _rpmostree_jsonutil_array_require_string_element (initramfs_args, i, error);
           if (!arg)
@@ -212,12 +350,37 @@ do_kernel_prep (int            rootfs_dfd,
                              cancellable, error))
     return FALSE;
 
-  if (!rpmostree_finalize_kernel (rootfs_dfd, bootdir, kver,
-                                  kernel_path,
+  /* We always tell rpmostree_finalize_kernel() to skip /boot, since we'll do a
+   * full hardlink pass if needed after that for the kernel + bootloader data.
+   */
+  if (!rpmostree_finalize_kernel (rootfs_dfd, bootdir, kver, kernel_path,
                                   &initramfs_tmpf,
+                                  RPMOSTREE_FINALIZE_KERNEL_USRLIB_OSTREEBOOT,
                                   cancellable, error))
     return FALSE;
 
+  /* We always ensure this exists as a mountpoint */
+  if (!glnx_ensure_dir (rootfs_dfd, "boot", 0755, error))
+    return FALSE;
+
+  /* If the boot location includes /boot, we also need to copy /usr/lib/ostree-boot there */
+  switch (boot_location)
+    {
+    case RPMOSTREE_POSTPROCESS_BOOT_LOCATION_BOTH:
+      {
+        g_print ("Using boot location: both\n");
+        /* Hardlink the existing content, only a little ugly as
+         * we'll end up sha256'ing it twice, but oh well. */
+        if (!hardlink_recurse (rootfs_dfd, "usr/lib/ostree-boot",
+                               rootfs_dfd, "boot",
+                               cancellable, error))
+          return glnx_prefix_error (error, "hardlinking /boot");
+      }
+      break;
+    case RPMOSTREE_POSTPROCESS_BOOT_LOCATION_NEW:
+      break;
+    }
+
   return TRUE;
 }
 
@@ -650,60 +813,6 @@ postprocess_selinux_policy_store_location (int rootfs_dfd,
   return TRUE;
 }
 
-static gboolean
-hardlink_recurse (int                src_dfd,
-                  const char        *src_path,
-                  int                dest_dfd,
-                  const char        *dest_path,
-                  GCancellable      *cancellable,
-                  GError            **error)
-{
-  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
-  glnx_fd_close int dest_target_dfd = -1;
-
-  if (!glnx_dirfd_iterator_init_at (src_dfd, src_path, TRUE, &dfd_iter, error))
-    return FALSE;
-
-  if (!glnx_opendirat (dest_dfd, dest_path, TRUE, &dest_target_dfd, error))
-    return FALSE;
-
-  while (TRUE)
-    {
-      struct dirent *dent = NULL;
-      struct stat stbuf;
-
-      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
-        return FALSE;
-      if (!dent)
-        break;
-
-      if (!glnx_fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW, error))
-        return FALSE;
-
-      if (dent->d_type == DT_DIR)
-        {
-          mode_t perms = stbuf.st_mode & ~S_IFMT;
-
-          if (!glnx_ensure_dir (dest_target_dfd, dent->d_name, perms, error))
-            return FALSE;
-          if (fchmodat (dest_target_dfd, dent->d_name, perms, 0) < 0)
-            return glnx_throw_errno_prefix (error, "fchmodat");
-          if (!hardlink_recurse (dfd_iter.fd, dent->d_name,
-                                 dest_target_dfd, dent->d_name,
-                                 cancellable, error))
-            return FALSE;
-        }
-      else
-        {
-          if (linkat (dfd_iter.fd, dent->d_name,
-                      dest_target_dfd, dent->d_name, 0) < 0)
-            return glnx_throw_errno_prefix (error, "linkat");
-        }
-    }
-
-  return TRUE;
-}
-
 /* Prepare a root filesystem, taking mainly the contents of /usr from pkgroot */
 static gboolean
 create_rootfs_from_pkgroot_content (int            target_root_dfd,
@@ -726,10 +835,7 @@ create_rootfs_from_pkgroot_content (int            target_root_dfd,
                                                                error))
     return FALSE;
 
-  g_print ("Preparing kernel\n");
-  if (!container && !do_kernel_prep (src_rootfs_fd, treefile, cancellable, error))
-    return glnx_prefix_error (error, "During kernel processing");
-
+  /* Initialize target root */
   g_print ("Initializing rootfs\n");
   gboolean tmp_is_dir = FALSE;
   if (!_rpmostree_jsonutil_object_get_optional_boolean_member (treefile,
@@ -784,65 +890,6 @@ create_rootfs_from_pkgroot_content (int            target_root_dfd,
   if (!convert_var_to_tmpfiles_d (src_rootfs_fd, target_root_dfd, cancellable, error))
     return FALSE;
 
-  /* Move boot, but rename the kernel/initramfs to have a checksum */
-  if (!container)
-    {
-      RpmOstreePostprocessBootLocation boot_location =
-        RPMOSTREE_POSTPROCESS_BOOT_LOCATION_BOTH;
-      const char *boot_location_str = NULL;
-
-      g_print ("Moving /boot\n");
-
-      if (!_rpmostree_jsonutil_object_get_optional_string_member (treefile,
-                                                                  "boot_location",
-                                                                  &boot_location_str, error))
-        return FALSE;
-
-      if (boot_location_str != NULL)
-        {
-          /* Note that "legacy" is now an alias for "both" */
-          if (strcmp (boot_location_str, "both") == 0 ||
-              strcmp (boot_location_str, "legacy") == 0)
-            boot_location = RPMOSTREE_POSTPROCESS_BOOT_LOCATION_BOTH;
-          else if (strcmp (boot_location_str, "new") == 0)
-            boot_location = RPMOSTREE_POSTPROCESS_BOOT_LOCATION_NEW;
-          else
-            return glnx_throw (error, "Invalid boot location '%s'", boot_location_str);
-        }
-
-      if (!glnx_shutil_mkdir_p_at (target_root_dfd, "usr/lib", 0755,
-                                   cancellable, error))
-        return FALSE;
-
-      switch (boot_location)
-        {
-        case RPMOSTREE_POSTPROCESS_BOOT_LOCATION_BOTH:
-          {
-            g_print ("Using boot location: both\n");
-            if (!glnx_renameat (src_rootfs_fd, "boot", target_root_dfd, "boot", error))
-              return FALSE;
-            if (!glnx_shutil_mkdir_p_at (target_root_dfd, "usr/lib/ostree-boot", 0755,
-                                         cancellable, error))
-              return FALSE;
-            /* Hardlink the existing content, only a little ugly as
-             * we'll end up sha256'ing it twice, but oh well. */
-            if (!hardlink_recurse (target_root_dfd, "boot",
-                                   target_root_dfd, "usr/lib/ostree-boot",
-                                   cancellable, error))
-              return FALSE;
-          }
-          break;
-        case RPMOSTREE_POSTPROCESS_BOOT_LOCATION_NEW:
-          {
-            g_print ("Using boot location: new\n");
-            if (!glnx_renameat (src_rootfs_fd, "boot",
-                                target_root_dfd, "usr/lib/ostree-boot", error))
-              return FALSE;
-          }
-          break;
-        }
-    }
-
   /* Also carry along toplevel compat links */
   g_print ("Copying toplevel compat symlinks\n");
   {
@@ -884,6 +931,27 @@ create_rootfs_from_pkgroot_content (int            target_root_dfd,
                           cancellable, error))
     return FALSE;
 
+  /* Handle kernel/initramfs if we're not doing a container */
+  if (!container)
+    {
+      g_print ("Preparing kernel\n");
+
+      /* OSTree needs to own this */
+      if (!glnx_shutil_rm_rf_at (src_rootfs_fd, "boot/loader", cancellable, error))
+        return FALSE;
+
+      /* The kernel may be in the source rootfs /boot; to handle that, we always
+       * rename the source /boot to the target, and will handle everything after
+       * that in the target root.
+       */
+      if (!glnx_renameat (src_rootfs_fd, "boot", target_root_dfd, "boot", error))
+        return FALSE;
+
+      if (!process_kernel_and_initramfs (target_root_dfd, treefile,
+                                         cancellable, error))
+        return glnx_prefix_error (error, "During kernel processing");
+    }
+
   return TRUE;
 }
 

tests/compose-tests/test-basic.sh

@@ -30,10 +30,14 @@ ostree --repo=${repobuild} show --print-metadata-key exampleos.tests ${treeref}
 assert_file_has_content meta.txt 'smoketested.*e2e'
 echo "ok metadata"
 
-ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/ostree-boot > bootls.txt
-assert_file_has_content bootls.txt vmlinuz
-assert_file_has_content bootls.txt initramfs
-echo "ok boot files"
+for path in /boot /usr/lib/ostree-boot; do
+    ostree --repo=${repobuild} ls -R ${treeref} ${path} > bootls.txt
+    assert_file_has_content bootls.txt vmlinuz-
+    assert_file_has_content bootls.txt initramfs-
+    echo "ok boot files"
+done
+kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-e].*$,\1,')
+ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null
 
 ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt
 assert_file_has_content manpages.txt man5/ostree.repo.5

tests/compose-tests/test-boot-location-new.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -xeuo pipefail
+
+dn=$(cd $(dirname $0) && pwd)
+. ${dn}/libcomposetest.sh
+
+prepare_compose_test "bootlocation-new"
+pysetjsonmember "boot_location" '"new"'
+runcompose
+echo "ok compose"
+
+# Nothing in /boot (but it should exist)
+ostree --repo=${repobuild} ls -R ${treeref} /boot > bootls.txt
+cat >bootls-expected.txt <<EOF
+d00755 0 0      0 /boot
+EOF
+diff -u bootls{-expected,}.txt
+# Verify /usr/lib/ostree-boot
+ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/ostree-boot > bootls.txt
+assert_file_has_content bootls.txt vmlinuz-
+assert_file_has_content bootls.txt initramfs-
+kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-e].*$,\1,')
+# And use the kver to find the kernel in /usr/lib/modules
+ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null
+echo "ok boot location new"