upgrader: Reuse existing rpmdb checkout if available

Check if we can reuse the base rpmdb from the pending deployment if it
matches the base rev we're targeting. This allows us to avoid checking
out the tree early only to later on discard it. Such cases include
layering existing packages and inactive requests.

Closes: #1502
Approved by: cgwalters

src/daemon/rpmostree-sysroot-upgrader.c

@@ -513,7 +513,8 @@ checkout_base_tree (RpmOstreeSysrootUpgrader *self,
                     GCancellable          *cancellable,
                     GError               **error)
 {
-  g_assert_cmpint (self->tmprootfs_dfd, ==, -1);
+  if (self->tmprootfs_dfd != -1)
+    return TRUE; /* already checked out! */
 
   /* let's give the user some feedback so they don't think we're blocked */
   rpmostree_output_task_begin ("Checking out tree %.7s", self->base_revision);
@@ -557,13 +558,78 @@ checkout_base_tree (RpmOstreeSysrootUpgrader *self,
                        &self->tmprootfs_dfd, error))
     return FALSE;
 
-  /* build a centralized rsack for it, since we need it in a few places */
-  self->rsack = rpmostree_get_refsack_for_root (self->tmprootfs_dfd, ".", error);
-  if (self->rsack == NULL)
+  rpmostree_output_task_end ("done");
+  return TRUE;
+}
+
+/* Optimization: use the already checked out base rpmdb of the pending deployment if the
+ * base layer matches. Returns FALSE on error, TRUE otherwise. Check self->rsack to
+ * determine if it worked. */
+static gboolean
+try_load_base_rsack_from_pending (RpmOstreeSysrootUpgrader *self,
+                                  GCancellable             *cancellable,
+                                  GError                  **error)
+{
+  gboolean is_live;
+  if (!rpmostree_syscore_deployment_is_live (self->sysroot, self->origin_merge_deployment,
+                                             &is_live, error))
     return FALSE;
 
-  rpmostree_output_task_end ("done");
+  /* livefs invalidates the deployment */
+  if (is_live)
+    return TRUE;
 
+  guint layer_version;
+  g_autofree char *base_rev_owned = NULL;
+  if (!rpmostree_deployment_get_layered_info (self->repo, self->origin_merge_deployment,
+                                              NULL, &layer_version, &base_rev_owned, NULL,
+                                              NULL, NULL, error))
+    return FALSE;
+
+  /* older client layers have a bug blocking us from using their base rpmdb:
+   * https://github.com/projectatomic/rpm-ostree/pull/1560 */
+  if (base_rev_owned && layer_version < 4)
+    return TRUE;
+
+  const char *base_rev =
+    base_rev_owned ?: ostree_deployment_get_csum (self->origin_merge_deployment);
+
+  /* it's no longer the base layer we're looking for (e.g. likely pulled a fresh one) */
+  if (!g_str_equal (self->base_revision, base_rev))
+    return TRUE;
+
+  int sysroot_fd = ostree_sysroot_get_fd (self->sysroot);
+  g_autofree char *path =
+    ostree_sysroot_get_deployment_dirpath (self->sysroot, self->origin_merge_deployment);
+
+  /* this may not actually populate the rsack if it's an old deployment */
+  if (!rpmostree_get_base_refsack_for_root (sysroot_fd, path, &self->rsack,
+                                            cancellable, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+static gboolean
+load_base_rsack (RpmOstreeSysrootUpgrader *self,
+                 GCancellable             *cancellable,
+                 GError                  **error)
+{
+  if (!try_load_base_rsack_from_pending (self, cancellable, error))
+    return FALSE;
+
+  if (self->rsack == NULL)
+    {
+      /* fallback to checking out the tree early; will be reused later for assembly */
+      if (!checkout_base_tree (self, cancellable, error))
+        return FALSE;
+
+      self->rsack = rpmostree_get_refsack_for_root (self->tmprootfs_dfd, ".", error);
+      if (self->rsack == NULL)
+        return FALSE;
+    }
+
+  g_assert (self->rsack != NULL);
   return TRUE;
 }
 
@@ -632,6 +698,8 @@ finalize_removal_overrides (RpmOstreeSysrootUpgrader *self,
                             GCancellable             *cancellable,
                             GError                  **error)
 {
+  g_assert (self->rsack);
+
   GHashTable *removals = rpmostree_origin_get_overrides_remove (self->origin);
   g_autoptr(GPtrArray) ret_final_removals = g_ptr_array_new_with_free_func (g_free);
 
@@ -665,6 +733,8 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self,
                                 GCancellable             *cancellable,
                                 GError                  **error)
 {
+  g_assert (self->rsack);
+
   GHashTable *local_replacements =
     rpmostree_origin_get_overrides_local_replace (self->origin);
   g_autoptr(GPtrArray) ret_final_local_replacements =
@@ -727,6 +797,8 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self,
                    GCancellable             *cancellable,
                    GError                  **error)
 {
+  g_assert (self->rsack);
+
   /* request (owned by origin) --> providing nevra */
   g_autoptr(GHashTable) inactive_requests =
     g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free);
@@ -854,6 +926,10 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self,
                      GError                  **error)
 {
   g_assert (!self->ctx);
+
+  if (!checkout_base_tree (self, cancellable, error))
+    return FALSE;
+
   self->ctx = rpmostree_context_new_system (self->repo, cancellable, error);
   g_autofree char *tmprootfs_abspath = glnx_fdrel_abspath (self->tmprootfs_dfd, ".");
 
@@ -1083,7 +1159,7 @@ rpmostree_sysroot_upgrader_prep_layering (RpmOstreeSysrootUpgrader *self,
     }
 
   /* Do a bit more work to see whether or not we have to do assembly */
-  if (!checkout_base_tree (self, cancellable, error))
+  if (!load_base_rsack (self, cancellable, error))
     return FALSE;
   if (!finalize_overrides (self, cancellable, error))
     return FALSE;

src/libpriv/rpmostree-origin.c

@@ -334,7 +334,10 @@ rpmostree_origin_get_unconfigured_state (RpmOstreeOrigin *origin)
 
 /* Determines whether the origin hints at local assembly being required. In some
  * cases, no assembly might actually be required (e.g. if requested packages are
- * already in the base). */
+ * already in the base). IOW:
+ *    FALSE --> definitely does not require local assembly
+ *    TRUE  --> maybe requires assembly, need to investigate further by doing work
+ */
 gboolean
 rpmostree_origin_may_require_local_assembly (RpmOstreeOrigin *origin)
 {

src/libpriv/rpmostree-rpm-util.c

@@ -836,6 +836,43 @@ rpmostree_get_refsack_for_root (int              dfd,
   return rpmostree_refsack_new (sack, NULL);
 }
 
+/* Given @dfd + @path, return a sack corresponding to the base layer (which is the same as
+ * /usr/share/rpm if it's not a layered deployment.
+ *
+ * Note this function may return %TRUE without a sack if the deployment predates when we
+ * started embedding RPMOSTREE_BASE_RPMDB.
+ */
+gboolean
+rpmostree_get_base_refsack_for_root (int                dfd,
+                                     const char        *path,
+                                     RpmOstreeRefSack **out_sack,
+                                     GCancellable      *cancellable,
+                                     GError           **error)
+{
+  g_autofree char *subpath = g_build_filename (path, RPMOSTREE_BASE_RPMDB, NULL);
+  if (!glnx_fstatat_allow_noent (dfd, subpath, NULL, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  if (errno == ENOENT)
+    return TRUE;
+
+  g_auto(GLnxTmpDir) tmpdir = {0, };
+  if (!glnx_mkdtemp ("rpmostree-dbquery-XXXXXX", 0700, &tmpdir, error))
+    return FALSE;
+  if (!glnx_shutil_mkdir_p_at (tmpdir.fd, "var/lib", 0777, cancellable, error))
+    return FALSE;
+
+  g_autofree char *base_rpm = glnx_fdrel_abspath (dfd, subpath);
+  if (symlinkat (base_rpm, tmpdir.fd, "var/lib/rpm") == -1)
+    return glnx_throw_errno_prefix (error, "symlinkat");
+
+  g_autoptr(DnfSack) sack = NULL; /* NB: refsack adds a ref to it */
+  if (!get_sack_for_root (tmpdir.fd, ".", &sack, error))
+    return FALSE;
+
+  *out_sack = rpmostree_refsack_new (sack, &tmpdir);
+  return TRUE;
+}
+
 
 /* Given @ref which is an OSTree ref, return a "sack" i.e. database of packages.
  */

src/libpriv/rpmostree-rpm-util.h

@@ -107,6 +107,13 @@ rpmostree_get_refsack_for_root (int              dfd,
                                 const char      *path,
                                 GError         **error);
 
+gboolean
+rpmostree_get_base_refsack_for_root (int                dfd,
+                                     const char        *path,
+                                     RpmOstreeRefSack **out_sack,
+                                     GCancellable      *cancellable,
+                                     GError           **error);
+
 RpmOstreeRefSack *
 rpmostree_get_base_refsack_for_commit (OstreeRepo                *repo,
                                        const char                *ref,

tests/vmcheck/test-layering-basic-2.sh

@@ -28,6 +28,8 @@ set -x
 vm_build_rpm foo
 vm_rpmostree install foo | tee output.txt
 assert_file_has_content output.txt '^Importing (1/1)'
+# also check that we definitely had to checkout the tree
+assert_file_has_content output.txt "Checking out tree "
 
 # upgrade with same foo in repos --> shouldn't re-import
 vm_cmd ostree commit -b vmcheck --tree=ref=vmcheck
@@ -89,18 +91,24 @@ vm_cmd ostree show --print-metadata-key rpmostree.rpmdb.pkglist \
 assert_file_has_content pkglist.txt 'test-pkgcache-migrate-pkg'
 echo "ok layered pkglist"
 
+# remove accumulated crud from previous tests
+vm_rpmostree uninstall --all
+vm_reboot
+
 if vm_rpmostree install glibc &>out.txt; then
   assert_not_reached "Successfully requested glibc without --allow-inactive?"
 fi
 assert_file_has_content out.txt "Use --allow-inactive to explicitly require it."
 vm_rpmostree cleanup -p
 vm_rpmostree install glibc --allow-inactive &>out.txt
+# if we have a base rpmdb, then we should be able to figure out it's inactive
+# without checking out the tree -- this conditional is needed because of CentOS
+if vm_cmd test -d /usr/lib/sysimage/rpm-ostree-base-db; then
+  assert_not_file_has_content out.txt "Checking out tree "
+fi
 vm_rpmostree cleanup -p
 echo "ok --allow-inactive"
 
-# remove accumulated crud from previous tests
-vm_rpmostree uninstall --all
-vm_reboot
 vm_rpmostree uninstall --all |& tee out.txt
 assert_file_has_content out.txt "No change."
 vm_build_rpm test-uninstall-all-pkg1