From a75460f538c60d5e3daf2f2a5c8c12bd5c423b1a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 13 Sep 2018 17:35:37 -0400 Subject: [PATCH] upgrader: Reuse existing rpmdb checkout if available Check if we can reuse the base rpmdb from the pending deployment if it matches the base rev we're targeting. This allows us to avoid checking out the tree early only to later on discard it. Such cases include layering existing packages and inactive requests. Closes: #1502 Approved by: cgwalters --- src/daemon/rpmostree-sysroot-upgrader.c | 88 +++++++++++++++++++++++-- src/libpriv/rpmostree-origin.c | 5 +- src/libpriv/rpmostree-rpm-util.c | 37 +++++++++++ src/libpriv/rpmostree-rpm-util.h | 7 ++ tests/vmcheck/test-layering-basic-2.sh | 14 +++- 5 files changed, 141 insertions(+), 10 deletions(-) diff --git a/src/daemon/rpmostree-sysroot-upgrader.c b/src/daemon/rpmostree-sysroot-upgrader.c index 8bad433b..76c56984 100644 --- a/src/daemon/rpmostree-sysroot-upgrader.c +++ b/src/daemon/rpmostree-sysroot-upgrader.c @@ -513,7 +513,8 @@ checkout_base_tree (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { - g_assert_cmpint (self->tmprootfs_dfd, ==, -1); + if (self->tmprootfs_dfd != -1) + return TRUE; /* already checked out! */ /* let's give the user some feedback so they don't think we're blocked */ rpmostree_output_task_begin ("Checking out tree %.7s", self->base_revision); @@ -557,13 +558,78 @@ checkout_base_tree (RpmOstreeSysrootUpgrader *self, &self->tmprootfs_dfd, error)) return FALSE; - /* build a centralized rsack for it, since we need it in a few places */ - self->rsack = rpmostree_get_refsack_for_root (self->tmprootfs_dfd, ".", error); - if (self->rsack == NULL) + rpmostree_output_task_end ("done"); + return TRUE; +} + +/* Optimization: use the already checked out base rpmdb of the pending deployment if the + * base layer matches. Returns FALSE on error, TRUE otherwise. Check self->rsack to + * determine if it worked. */ +static gboolean +try_load_base_rsack_from_pending (RpmOstreeSysrootUpgrader *self, + GCancellable *cancellable, + GError **error) +{ + gboolean is_live; + if (!rpmostree_syscore_deployment_is_live (self->sysroot, self->origin_merge_deployment, + &is_live, error)) return FALSE; - rpmostree_output_task_end ("done"); + /* livefs invalidates the deployment */ + if (is_live) + return TRUE; + guint layer_version; + g_autofree char *base_rev_owned = NULL; + if (!rpmostree_deployment_get_layered_info (self->repo, self->origin_merge_deployment, + NULL, &layer_version, &base_rev_owned, NULL, + NULL, NULL, error)) + return FALSE; + + /* older client layers have a bug blocking us from using their base rpmdb: + * https://github.com/projectatomic/rpm-ostree/pull/1560 */ + if (base_rev_owned && layer_version < 4) + return TRUE; + + const char *base_rev = + base_rev_owned ?: ostree_deployment_get_csum (self->origin_merge_deployment); + + /* it's no longer the base layer we're looking for (e.g. likely pulled a fresh one) */ + if (!g_str_equal (self->base_revision, base_rev)) + return TRUE; + + int sysroot_fd = ostree_sysroot_get_fd (self->sysroot); + g_autofree char *path = + ostree_sysroot_get_deployment_dirpath (self->sysroot, self->origin_merge_deployment); + + /* this may not actually populate the rsack if it's an old deployment */ + if (!rpmostree_get_base_refsack_for_root (sysroot_fd, path, &self->rsack, + cancellable, error)) + return FALSE; + + return TRUE; +} + +static gboolean +load_base_rsack (RpmOstreeSysrootUpgrader *self, + GCancellable *cancellable, + GError **error) +{ + if (!try_load_base_rsack_from_pending (self, cancellable, error)) + return FALSE; + + if (self->rsack == NULL) + { + /* fallback to checking out the tree early; will be reused later for assembly */ + if (!checkout_base_tree (self, cancellable, error)) + return FALSE; + + self->rsack = rpmostree_get_refsack_for_root (self->tmprootfs_dfd, ".", error); + if (self->rsack == NULL) + return FALSE; + } + + g_assert (self->rsack != NULL); return TRUE; } @@ -632,6 +698,8 @@ finalize_removal_overrides (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { + g_assert (self->rsack); + GHashTable *removals = rpmostree_origin_get_overrides_remove (self->origin); g_autoptr(GPtrArray) ret_final_removals = g_ptr_array_new_with_free_func (g_free); @@ -665,6 +733,8 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { + g_assert (self->rsack); + GHashTable *local_replacements = rpmostree_origin_get_overrides_local_replace (self->origin); g_autoptr(GPtrArray) ret_final_local_replacements = @@ -727,6 +797,8 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { + g_assert (self->rsack); + /* request (owned by origin) --> providing nevra */ g_autoptr(GHashTable) inactive_requests = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free); @@ -854,6 +926,10 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GError **error) { g_assert (!self->ctx); + + if (!checkout_base_tree (self, cancellable, error)) + return FALSE; + self->ctx = rpmostree_context_new_system (self->repo, cancellable, error); g_autofree char *tmprootfs_abspath = glnx_fdrel_abspath (self->tmprootfs_dfd, "."); @@ -1083,7 +1159,7 @@ rpmostree_sysroot_upgrader_prep_layering (RpmOstreeSysrootUpgrader *self, } /* Do a bit more work to see whether or not we have to do assembly */ - if (!checkout_base_tree (self, cancellable, error)) + if (!load_base_rsack (self, cancellable, error)) return FALSE; if (!finalize_overrides (self, cancellable, error)) return FALSE; diff --git a/src/libpriv/rpmostree-origin.c b/src/libpriv/rpmostree-origin.c index 0ed49694..b2be1df0 100644 --- a/src/libpriv/rpmostree-origin.c +++ b/src/libpriv/rpmostree-origin.c @@ -334,7 +334,10 @@ rpmostree_origin_get_unconfigured_state (RpmOstreeOrigin *origin) /* Determines whether the origin hints at local assembly being required. In some * cases, no assembly might actually be required (e.g. if requested packages are - * already in the base). */ + * already in the base). IOW: + * FALSE --> definitely does not require local assembly + * TRUE --> maybe requires assembly, need to investigate further by doing work + */ gboolean rpmostree_origin_may_require_local_assembly (RpmOstreeOrigin *origin) { diff --git a/src/libpriv/rpmostree-rpm-util.c b/src/libpriv/rpmostree-rpm-util.c index 23f2c217..edd2ed8d 100644 --- a/src/libpriv/rpmostree-rpm-util.c +++ b/src/libpriv/rpmostree-rpm-util.c @@ -836,6 +836,43 @@ rpmostree_get_refsack_for_root (int dfd, return rpmostree_refsack_new (sack, NULL); } +/* Given @dfd + @path, return a sack corresponding to the base layer (which is the same as + * /usr/share/rpm if it's not a layered deployment. + * + * Note this function may return %TRUE without a sack if the deployment predates when we + * started embedding RPMOSTREE_BASE_RPMDB. + */ +gboolean +rpmostree_get_base_refsack_for_root (int dfd, + const char *path, + RpmOstreeRefSack **out_sack, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *subpath = g_build_filename (path, RPMOSTREE_BASE_RPMDB, NULL); + if (!glnx_fstatat_allow_noent (dfd, subpath, NULL, AT_SYMLINK_NOFOLLOW, error)) + return FALSE; + if (errno == ENOENT) + return TRUE; + + g_auto(GLnxTmpDir) tmpdir = {0, }; + if (!glnx_mkdtemp ("rpmostree-dbquery-XXXXXX", 0700, &tmpdir, error)) + return FALSE; + if (!glnx_shutil_mkdir_p_at (tmpdir.fd, "var/lib", 0777, cancellable, error)) + return FALSE; + + g_autofree char *base_rpm = glnx_fdrel_abspath (dfd, subpath); + if (symlinkat (base_rpm, tmpdir.fd, "var/lib/rpm") == -1) + return glnx_throw_errno_prefix (error, "symlinkat"); + + g_autoptr(DnfSack) sack = NULL; /* NB: refsack adds a ref to it */ + if (!get_sack_for_root (tmpdir.fd, ".", &sack, error)) + return FALSE; + + *out_sack = rpmostree_refsack_new (sack, &tmpdir); + return TRUE; +} + /* Given @ref which is an OSTree ref, return a "sack" i.e. database of packages. */ diff --git a/src/libpriv/rpmostree-rpm-util.h b/src/libpriv/rpmostree-rpm-util.h index c73d3946..c7ffa373 100644 --- a/src/libpriv/rpmostree-rpm-util.h +++ b/src/libpriv/rpmostree-rpm-util.h @@ -107,6 +107,13 @@ rpmostree_get_refsack_for_root (int dfd, const char *path, GError **error); +gboolean +rpmostree_get_base_refsack_for_root (int dfd, + const char *path, + RpmOstreeRefSack **out_sack, + GCancellable *cancellable, + GError **error); + RpmOstreeRefSack * rpmostree_get_base_refsack_for_commit (OstreeRepo *repo, const char *ref, diff --git a/tests/vmcheck/test-layering-basic-2.sh b/tests/vmcheck/test-layering-basic-2.sh index eb2aca46..50e326eb 100755 --- a/tests/vmcheck/test-layering-basic-2.sh +++ b/tests/vmcheck/test-layering-basic-2.sh @@ -28,6 +28,8 @@ set -x vm_build_rpm foo vm_rpmostree install foo | tee output.txt assert_file_has_content output.txt '^Importing (1/1)' +# also check that we definitely had to checkout the tree +assert_file_has_content output.txt "Checking out tree " # upgrade with same foo in repos --> shouldn't re-import vm_cmd ostree commit -b vmcheck --tree=ref=vmcheck @@ -89,18 +91,24 @@ vm_cmd ostree show --print-metadata-key rpmostree.rpmdb.pkglist \ assert_file_has_content pkglist.txt 'test-pkgcache-migrate-pkg' echo "ok layered pkglist" +# remove accumulated crud from previous tests +vm_rpmostree uninstall --all +vm_reboot + if vm_rpmostree install glibc &>out.txt; then assert_not_reached "Successfully requested glibc without --allow-inactive?" fi assert_file_has_content out.txt "Use --allow-inactive to explicitly require it." vm_rpmostree cleanup -p vm_rpmostree install glibc --allow-inactive &>out.txt +# if we have a base rpmdb, then we should be able to figure out it's inactive +# without checking out the tree -- this conditional is needed because of CentOS +if vm_cmd test -d /usr/lib/sysimage/rpm-ostree-base-db; then + assert_not_file_has_content out.txt "Checking out tree " +fi vm_rpmostree cleanup -p echo "ok --allow-inactive" -# remove accumulated crud from previous tests -vm_rpmostree uninstall --all -vm_reboot vm_rpmostree uninstall --all |& tee out.txt assert_file_has_content out.txt "No change." vm_build_rpm test-uninstall-all-pkg1