daemon: Unify pkgcache with system repo

We originally needed the pkgcache to be a separate repo due to ostree's
overzealous pruning policies. The idea was to maintain multiple commits
in each pkg branch for different SELinux policies. In practice, there's
not much use in maintaining old copies and it's just easier to always
relabel on the fly. So then, the need for a separate repo completely
melts away.

This helps simplify the mental model a bit and allows us to avoid subtle
issues like #1047. Note however that the core is still capable of
handling split repos for the `--ex-unified-core` compose use case. Once
that and the jigdo work are a bit more settled, we can have a clearer
picture of how to simplify the core further.

The tricky bit is migrating the cache. When deploying, we check if a
pkgcache repo exists and migrate its refs if so. We then leave behind a
symlink to the system repo to remain compatible with older rpm-ostrees.

Closes: #1055
Approved by: cgwalters

src/daemon/rpmostree-sysroot-core.c

@@ -180,10 +180,6 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
   g_autoptr(GHashTable) referenced_pkgs = /* cache refs of packages we want to keep */
     g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
 
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
   g_autoptr(GPtrArray) deployments = ostree_sysroot_get_deployments (sysroot);
   for (guint i = 0; i < deployments->len; i++)
     {
@@ -219,7 +215,7 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
       GLNX_HASH_TABLE_FOREACH (local_replace, const char*, nevra)
         {
           g_autofree char *cachebranch = NULL;
-          if (!rpmostree_find_cache_branch_by_nevra (pkgcache_repo, nevra, &cachebranch,
+          if (!rpmostree_find_cache_branch_by_nevra (repo, nevra, &cachebranch,
                                                      cancellable, error))
             return FALSE;
 
@@ -228,7 +224,7 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
     }
 
   g_autoptr(GHashTable) current_refs = NULL;
-  if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", &current_refs,
+  if (!ostree_repo_list_refs_ext (repo, "rpmostree/pkg", &current_refs,
                                   OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
     return FALSE;
 
@@ -238,15 +234,17 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
       if (g_hash_table_contains (referenced_pkgs, ref))
         continue;
 
-      if (!ostree_repo_set_ref_immediate (pkgcache_repo, NULL, ref, NULL,
+      if (!ostree_repo_set_ref_immediate (repo, NULL, ref, NULL,
                                           cancellable, error))
         return FALSE;
       n_freed++;
     }
 
+  /* note that we're called right after an ostree_sysroot_cleanup(), so the stats reported
+   * accurately reflect pkgcache branches only */
   guint64 freed_space;
   gint n_objects_total, n_objects_pruned;
-  if (!ostree_repo_prune (pkgcache_repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0,
+  if (!ostree_repo_prune (repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0,
                           &n_objects_total, &n_objects_pruned, &freed_space,
                           cancellable, error))
     return FALSE;

src/daemon/rpmostree-sysroot-upgrader.c

@@ -612,15 +612,12 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self,
     g_ptr_array_new_with_free_func (g_free);
 
   g_autoptr(GPtrArray) inactive_replacements = g_ptr_array_new ();
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
 
   GLNX_HASH_TABLE_FOREACH_KV (local_replacements, const char*, nevra, const char*, sha256)
     {
       /* use the pkgcache because there's no safe way to go from nevra --> pkgname */
       g_autofree char *pkgname = NULL;
-      if (!rpmostree_get_nevra_from_pkgcache (pkgcache_repo, nevra, &pkgname, NULL, NULL,
+      if (!rpmostree_get_nevra_from_pkgcache (self->repo, nevra, &pkgname, NULL, NULL,
                                               NULL, NULL, cancellable, error))
         return FALSE;
 
@@ -688,17 +685,13 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self,
       if (!initialize_metatmpdir (self, error))
         return FALSE;
 
-      g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-      if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-        return FALSE;
-
       GLNX_HASH_TABLE_FOREACH_KV (local_pkgs, const char*, nevra, const char*, sha256)
         {
           g_autoptr(GVariant) header = NULL;
           g_autofree char *path =
             g_strdup_printf ("%s/%s.rpm", self->metatmpdir.path, nevra);
 
-          if (!rpmostree_pkgcache_find_pkg_header (pkgcache_repo, nevra, sha256,
+          if (!rpmostree_pkgcache_find_pkg_header (self->repo, nevra, sha256,
                                                    &header, cancellable, error))
             return FALSE;
 
@@ -804,7 +797,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self,
                      GError                  **error)
 {
   g_assert (!self->ctx);
-  self->ctx = rpmostree_context_new_system (cancellable, error);
+  self->ctx = rpmostree_context_new_system (self->repo, cancellable, error);
   g_autofree char *tmprootfs_abspath = glnx_fdrel_abspath (self->tmprootfs_dfd, ".");
 
   if (!prepare_context_for_assembly (self, tmprootfs_abspath, cancellable, error))
@@ -823,12 +816,6 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self,
                                 cancellable, error))
     return FALSE;
 
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
-  rpmostree_context_set_repos (self->ctx, self->repo, pkgcache_repo);
-
   const gboolean have_packages = (self->overlay_packages->len > 0 ||
                                   g_hash_table_size (local_pkgs) > 0 ||
                                   self->override_remove_packages->len > 0 ||

src/daemon/rpmostreed-transaction-types.c

@@ -498,27 +498,20 @@ import_local_rpm (OstreeRepo    *repo,
 }
 
 static gboolean
-import_many_local_rpms (OstreeRepo    *parent,
+import_many_local_rpms (OstreeRepo    *repo,
                         GUnixFDList   *fdl,
                         GPtrArray    **out_pkgs,
                         GCancellable  *cancellable,
                         GError       **error)
 {
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-
-  /* It might seem risky to rely on the cache as the source of truth for local
-   * RPMs. However, the core will never re-import the same NEVRA if it's already
-   * present. To be safe, we do also record the SHA-256 of the RPM header in the
-   * origin. We don't record the checksum of the branch itself, because it may
-   * need relabeling and that's OK.
+  /* Note that we record the SHA-256 of the RPM header in the origin to make sure that e.g.
+   * if we somehow re-import the same NEVRA with different content, we error out. We don't
+   * record the checksum of the branch itself, because it may need relabeling and that's OK.
    * */
 
-  if (!rpmostree_syscore_get_pkgcache_repo (parent, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
   g_auto(RpmOstreeRepoAutoTransaction) txn = { 0, };
   /* Note use of commit-on-failure */
-  if (!rpmostree_repo_auto_transaction_start (&txn, pkgcache_repo, TRUE, cancellable, error))
+  if (!rpmostree_repo_auto_transaction_start (&txn, repo, TRUE, cancellable, error))
     return FALSE;
 
   g_autoptr(GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_free);
@@ -528,13 +521,13 @@ import_many_local_rpms (OstreeRepo    *parent,
   for (guint i = 0; i < nfds; i++)
     {
       g_autofree char *sha256_nevra = NULL;
-      if (!import_local_rpm (pkgcache_repo, fds[i], &sha256_nevra, cancellable, error))
+      if (!import_local_rpm (repo, fds[i], &sha256_nevra, cancellable, error))
         return FALSE;
 
       g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra));
     }
 
-  if (!ostree_repo_commit_transaction (pkgcache_repo, NULL, cancellable, error))
+  if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
     return FALSE;
   txn.initialized = FALSE;
 
@@ -598,6 +591,11 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
       upgrader_flags |= RPMOSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED;
     }
 
+  /* before doing any real work, let's make sure the pkgcache is migrated */
+  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
+  if (!rpmostree_migrate_pkgcache_repo (repo, cancellable, error))
+    return FALSE;
+
   g_autoptr(RpmOstreeSysrootUpgrader) upgrader =
     rpmostree_sysroot_upgrader_new (sysroot, self->osname, upgrader_flags,
                                     cancellable, error);
@@ -616,10 +614,6 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
         return FALSE;
     }
 
-  g_autoptr(OstreeRepo) repo = NULL;
-  if (!ostree_sysroot_get_repo (sysroot, &repo, cancellable, error))
-    return FALSE;
-
   g_autoptr(OstreeAsyncProgress) progress =
     ostree_async_progress_new ();
 
@@ -1360,7 +1354,8 @@ refresh_md_transaction_execute (RpmostreedTransaction *transaction,
   g_autofree char *origin_deployment_root =
     g_build_filename (sysroot_path, origin_deployment_dirpath, NULL);
 
-  g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (cancellable, error);
+  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
+  g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (repo, cancellable, error);
 
   /* We could bypass rpmostree_context_setup() here and call dnf_context_setup() ourselves
    * since we're not actually going to perform any installation. Though it does provide us

src/libpriv/rpmostree-core.c

@@ -321,10 +321,14 @@ set_rpm_macro_define (const char *key, const char *value)
 }
 
 RpmOstreeContext *
-rpmostree_context_new_system (GCancellable *cancellable,
+rpmostree_context_new_system (OstreeRepo   *repo,
+                              GCancellable *cancellable,
                               GError      **error)
 {
+  g_return_val_if_fail (repo != NULL, FALSE);
+
   RpmOstreeContext *self = g_object_new (RPMOSTREE_TYPE_CONTEXT, NULL);
+  self->ostreerepo = g_object_ref (repo);
 
   /* We can always be control-c'd at any time; this is new API,
    * otherwise we keep calling _rpmostree_reset_rpm_sighandlers() in
@@ -366,8 +370,7 @@ rpmostree_context_new_tree (int               userroot_dfd,
                             GCancellable     *cancellable,
                             GError          **error)
 {
-  g_autoptr(RpmOstreeContext) ret =
-    rpmostree_context_new_system (cancellable, error);
+  g_autoptr(RpmOstreeContext) ret = rpmostree_context_new_system (repo, cancellable, error);
   if (!ret)
     return NULL;
 
@@ -387,9 +390,6 @@ rpmostree_context_new_tree (int               userroot_dfd,
     dnf_context_set_lock_dir (ret->dnfctx, lockdir);
   }
 
-  /* open user root repo if exists (container path) */
-  ret->ostreerepo = repo ? g_object_ref (repo) : NULL;
-
   return g_steal_pointer (&ret);
 }
 
@@ -927,7 +927,7 @@ checkout_pkg_metadata_by_nevra (RpmOstreeContext *self,
 {
   g_autoptr(GVariant) header = NULL;
 
-  if (!rpmostree_pkgcache_find_pkg_header (self->pkgcache_repo, nevra, sha256,
+  if (!rpmostree_pkgcache_find_pkg_header (get_pkgcache_repo (self), nevra, sha256,
                                            &header, cancellable, error))
     return FALSE;
 
@@ -1653,14 +1653,15 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self,
                              GCancellable     *cancellable,
                              GError          **error)
 {
-  g_assert (self->pkgcache_repo);
+  OstreeRepo *pkgcache_repo = get_pkgcache_repo (self);
+  g_assert (pkgcache_repo);
   g_assert (self->pkgcache_only);
 
   DnfSack *sack = dnf_context_get_sack (self->dnfctx);
   g_assert (sack);
 
   g_autoptr(GHashTable) refs = NULL;
-  if (!ostree_repo_list_refs_ext (self->pkgcache_repo, "rpmostree/pkg", &refs,
+  if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", &refs,
                                   OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
     return FALSE;
 
@@ -1671,7 +1672,7 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self,
         continue;
 
       g_autoptr(GVariant) header = NULL;
-      if (!get_header_variant (self->pkgcache_repo, ref, &header, cancellable, error))
+      if (!get_header_variant (pkgcache_repo, ref, &header, cancellable, error))
         return FALSE;
 
       if (!checkout_pkg_metadata (self, nevra, header, cancellable, error))
@@ -2158,6 +2159,10 @@ checkout_package_into_root (RpmOstreeContext *self,
 {
   OstreeRepo *pkgcache_repo = get_pkgcache_repo (self);
 
+  /* The below is currently TRUE only in the --ex-unified-core path. We probably want to
+   * migrate that over to always use a separate cache repo eventually, which would allow us
+   * to completely drop the pkgcache_repo/ostreerepo dichotomy in the core. See:
+   * https://github.com/projectatomic/rpm-ostree/pull/1055 */
   if (pkgcache_repo != self->ostreerepo)
     {
       if (!rpmostree_pull_content_only (self->ostreerepo, pkgcache_repo, pkg_commit,

src/libpriv/rpmostree-core.h

@@ -36,8 +36,9 @@ G_DECLARE_FINAL_TYPE (RpmOstreeContext, rpmostree_context, RPMOSTREE, CONTEXT, G
 #define RPMOSTREE_TYPE_TREESPEC (rpmostree_treespec_get_type ())
 G_DECLARE_FINAL_TYPE (RpmOstreeTreespec, rpmostree_treespec, RPMOSTREE, TREESPEC, GObject)
 
-RpmOstreeContext *rpmostree_context_new_system (GCancellable *cancellable,
-                                                GError **error);
+RpmOstreeContext *rpmostree_context_new_system (OstreeRepo   *repo,
+                                                GCancellable *cancellable,
+                                                GError      **error);
 
 RpmOstreeContext *rpmostree_context_new_tree (int basedir_dfd,
                                               OstreeRepo  *repo,

src/libpriv/rpmostree-util.c

@@ -25,12 +25,16 @@
 #include <glib-unix.h>
 #include <json-glib/json-glib.h>
 #include <gio/gunixoutputstream.h>
+#include <systemd/sd-journal.h>
 
 #include "rpmostree-util.h"
 #include "rpmostree-origin.h"
+#include "rpmostree-output.h"
 #include "rpmostree.h"
 #include "libglnx.h"
 
+#define RPMOSTREE_OLD_PKGCACHE_DIR "extensions/rpmostree/pkgcache"
+
 int
 rpmostree_ptrarray_sort_compare_strings (gconstpointer ap,
                                          gconstpointer bp)
@@ -517,6 +521,94 @@ rpmostree_deployment_get_layered_info (OstreeRepo        *repo,
   return TRUE;
 }
 
+static gboolean
+do_pkgcache_migration (OstreeRepo    *repo,
+                       OstreeRepo    *pkgcache,
+                       guint         *out_n_migrated,
+                       GCancellable  *cancellable,
+                       GError       **error)
+{
+  g_autoptr(GHashTable) pkgcache_refs = NULL;
+  if (!ostree_repo_list_refs_ext (pkgcache, "rpmostree/pkg", &pkgcache_refs,
+                                  OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
+    return FALSE;
+
+  if (g_hash_table_size (pkgcache_refs) == 0)
+    {
+      *out_n_migrated = 0;
+      return TRUE; /* Note early return */
+    }
+
+  g_autofree char **refs_strv =
+    (char **)g_hash_table_get_keys_as_array (pkgcache_refs, NULL);
+
+  /* fd-relative pull API anyone? build the path manually at least to avoid one malloc */
+  g_autofree char *pkgcache_uri =
+    g_strdup_printf ("file:///proc/self/fd/%d", ostree_repo_get_dfd (pkgcache));
+  if (!ostree_repo_pull (repo, pkgcache_uri, refs_strv, OSTREE_REPO_PULL_FLAGS_NONE, NULL,
+                         cancellable, error))
+    return FALSE;
+
+  *out_n_migrated = g_strv_length (refs_strv);
+  return TRUE;
+}
+
+gboolean
+rpmostree_migrate_pkgcache_repo (OstreeRepo   *repo,
+                                 GCancellable *cancellable,
+                                 GError      **error)
+{
+  int repo_dfd = ostree_repo_get_dfd (repo);
+
+  struct stat stbuf;
+  if (!glnx_fstatat_allow_noent (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, &stbuf,
+                                 AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+
+  if (errno == 0 && S_ISLNK (stbuf.st_mode))
+    return TRUE;
+
+  /* if pkgcache exists, we expect it to be valid; we don't want to nuke it just because of
+   * a transient error */
+  if (errno == 0)
+    {
+      if (S_ISDIR (stbuf.st_mode))
+        {
+          rpmostree_output_task_begin ("Migrating pkgcache");
+
+          g_autoptr(OstreeRepo) pkgcache = ostree_repo_open_at (repo_dfd,
+                                                                RPMOSTREE_OLD_PKGCACHE_DIR,
+                                                                cancellable, error);
+          if (!pkgcache)
+            return FALSE;
+
+          guint n_migrated;
+          if (!do_pkgcache_migration (repo, pkgcache, &n_migrated, cancellable, error))
+            return FALSE;
+
+          rpmostree_output_task_end ("%u done", n_migrated);
+          if (n_migrated > 0)
+            sd_journal_print (LOG_INFO, "migrated %u cached package%s to system repo",
+                              n_migrated, _NS(n_migrated));
+        }
+
+      if (!glnx_shutil_rm_rf_at (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, cancellable, error))
+        return FALSE;
+    }
+  else
+    {
+      if (!glnx_shutil_mkdir_p_at (repo_dfd, dirname (strdupa (RPMOSTREE_OLD_PKGCACHE_DIR)),
+                                   0755, cancellable, error))
+        return FALSE;
+    }
+
+  /* leave a symlink for compatibility with older rpm-ostree versions */
+  if (symlinkat ("../..", repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR) < 0)
+    return glnx_throw_errno_prefix (error, "symlinkat");
+
+  return TRUE;
+}
+
 gboolean
 rpmostree_decompose_sha256_nevra (const char **nevra, /* gets incremented */
                                   char       **out_sha256,

src/libpriv/rpmostree-util.h

@@ -114,6 +114,11 @@ rpmostree_deployment_get_layered_info (OstreeRepo        *repo,
                                        GVariant         **out_replaced_base_pkgs,
                                        GError           **error);
 
+gboolean
+rpmostree_migrate_pkgcache_repo (OstreeRepo   *repo,
+                                 GCancellable *cancellable,
+                                 GError      **error);
+
 gboolean
 rpmostree_decompose_sha256_nevra (const char **nevra,
                                   char       **sha256,

tests/common/libvm.sh

@@ -84,7 +84,7 @@ vm_cmdfile() {
 
 # Delete anything which we might change between runs
 vm_clean_caches() {
-    vm_cmd rm /ostree/repo/extensions/rpmostree/pkgcache/refs/heads/* -rf
+    vm_cmd rm /ostree/repo/refs/heads/rpmostree/pkg/* -rf
 }
 
 # run rpm-ostree in vm
@@ -363,6 +363,8 @@ vm_get_journal_cursor() {
 }
 
 # Wait for a message logged after $cursor matching a regexp to appear
+# $1 - cursor
+# $2 - regex to wait for
 vm_wait_content_after_cursor() {
     from_cursor=$1; shift
     regex=$1; shift

tests/vmcheck/test-layering-basic.sh

@@ -61,10 +61,10 @@ echo "ok failed to install in /opt and /usr/local"
 
 vm_build_rpm foo
 vm_rpmostree pkg-add foo-1.0
-vm_cmd ostree --repo=/sysroot/ostree/repo/extensions/rpmostree/pkgcache refs |grep /foo/> refs.txt
+vm_cmd ostree refs |grep /foo/> refs.txt
 pkgref=$(head -1 refs.txt)
 # Verify we have a mapping from pkg-in-ostree → rpmmd-repo info
-vm_cmd ostree --repo=/sysroot/ostree/repo/extensions/rpmostree/pkgcache show --print-metadata-key rpmostree.repo ${pkgref} >refdata.txt
+vm_cmd ostree show --print-metadata-key rpmostree.repo ${pkgref} >refdata.txt
 assert_file_has_content refdata.txt 'id.*test-repo'
 assert_file_has_content refdata.txt 'timestamp'
 rm -f refs.txt refdata.txt
@@ -166,3 +166,27 @@ vm_cmd cat /usr/lib/ostree-boot/EFI/efi/fedora/fonts/unicode.pf2 > unicode.txt
 assert_file_has_content unicode.txt unicode
 echo "ok failed installed in /boot"
 
+# there should be a couple of pkgs already installed from the tests up above,
+# though let's add our own to be self-contained
+vm_build_rpm test-pkgcache-migrate-pkg1
+vm_build_rpm test-pkgcache-migrate-pkg2
+vm_rpmostree install test-pkgcache-migrate-pkg{1,2}
+
+# jury-rig a pkgcache of the olden days
+OLD_PKGCACHE_DIR=/ostree/repo/extensions/rpmostree/pkgcache
+vm_cmd test -L ${OLD_PKGCACHE_DIR}
+vm_cmd rm ${OLD_PKGCACHE_DIR}
+vm_cmd mkdir ${OLD_PKGCACHE_DIR}
+vm_cmd ostree init --repo ${OLD_PKGCACHE_DIR} --mode=bare
+vm_cmd ostree pull-local --repo ${OLD_PKGCACHE_DIR} /ostree/repo \
+  rpmostree/pkg/test-pkgcache-migrate-pkg{1,2}/1.0-1.x86__64
+vm_cmd ostree commit -b vmcheck --tree=ref=vmcheck
+cursor=$(vm_get_journal_cursor)
+vm_rpmostree upgrade | tee output.txt
+vm_wait_content_after_cursor $cursor 'migrated 2 cached packages'
+assert_file_has_content output.txt "Migrating pkgcache"
+for ref in rpmostree/pkg/test-pkgcache-migrate-pkg{1,2}/1.0-1.x86__64; do
+  vm_cmd ostree show $ref
+done
+vm_cmd test -L ${OLD_PKGCACHE_DIR}
+echo "ok migrate pkgcache"