manifest: Add lockfile-repos field

In Fedora CoreOS, we have a "coreos-pool" repo from which all packages
in lockfiles are tagged for reproducible builds. This repo is shared
across all streams, including those on f31 and f32.

Thus, it makes no sense for composes to ever pick packages unconstrained
from the pool without being guided by a lockfile. Otherwise, one can
easily end up with e.g. f32 packages in an f31 compose.

Add a new `lockfile-repos` for this which is only used for fetching
lockfile packages and nothing else. For example, this will allow
`cosa fetch --update-lockfile` to Just Work as expected by only fetching
new packages from regular yum repos.

docs/manual/treefile.md

@@ -17,7 +17,7 @@ It supports the following parameters:
    secret key must be in the home directory of the building user.  Defaults to
    none.
 
- * `repos` array of strings, mandatory: Names of yum repositories to
+ * `repos`: array of strings, mandatory: Names of yum repositories to
    use, from any files that end in `.repo`, in the same directory as
    the treefile.  `rpm-ostree compose tree` does not use the system
    `/etc/yum.repos.d`, because it's common to want to compose a target
@@ -304,3 +304,9 @@ version of `rpm-ostree`.
 
  * `rojig`: Object, optional.  Sub-keys are `name`, `summary`, `license`,
    and `description`.  Of those, `name` and `license` are mandatory.
+
+ * `lockfile-repos`: array of strings, optional: Semantically similar to
+   `repo`, but these repos will only be used to fetch packages locked
+   via lockfiles. This is useful when locked packages are kept
+   separately from the primary repos and one wants to ensure that
+   rpm-ostree will otherwise not select unlocked packages from them.

rust/src/treefile.rs

@@ -313,6 +313,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     );
     merge_vecs!(
         repos,
+        lockfile_repos,
         packages,
         bootstrap_packages,
         exclude_packages,
@@ -646,6 +647,9 @@ struct TreeComposeConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     repos: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "lockfile-repos")]
+    lockfile_repos: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     selinux: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(rename = "gpg-key")]

src/app/rpmostree-composeutil.c

@@ -251,6 +251,8 @@ rpmostree_composeutil_get_treespec (RpmOstreeContext  *ctx,
     return FALSE;
   if (!treespec_bind_array (treedata, treespec, "repos", NULL, TRUE, error))
     return FALSE;
+  if (!treespec_bind_array (treedata, treespec, "lockfile-repos", NULL, FALSE, error))
+    return FALSE;
   if (!treespec_bind_bool (treedata, treespec, "documentation", TRUE, error))
     return FALSE;
   if (!treespec_bind_bool (treedata, treespec, "recommends", TRUE, error))

src/libpriv/rpmostree-core.c

@@ -268,6 +268,10 @@ rpmostree_treespec_new_from_keyfile (GKeyFile   *keyfile,
     if (val && *val)
       add_canonicalized_string_array (&builder, "repos", NULL, keyfile);
   }
+  { g_auto(GStrv) val = g_key_file_get_string_list (keyfile, "tree", "lockfile-repos", NULL, NULL);
+    if (val && *val)
+      add_canonicalized_string_array (&builder, "lockfile-repos", NULL, keyfile);
+  }
   add_canonicalized_string_array (&builder, "instlangs", "instlangs-all", keyfile);
 
   if (g_key_file_get_boolean (keyfile, "tree", "skip-sanity-check", NULL))
@@ -758,15 +762,41 @@ rpmostree_context_setup (RpmOstreeContext    *self,
     }
   else
     {
+      /* Makes sure we only disable all repos once. This is more for future proofing against
+       * refactors for now since we don't support `lockfile-repos` on the client-side and on
+       * the server-side we always require `repos` anyway. */
+      gboolean disabled_all_repos = FALSE;
+
       /* NB: missing "repos" --> let libdnf figure it out for itself (we're likely doing a
        * client-side compose where we want to use /etc/yum.repos.d/) */
       g_autofree char **enabled_repos = NULL;
       if (g_variant_dict_lookup (self->spec->dict, "repos", "^a&s", &enabled_repos))
         {
-          disable_all_repos (self);
+          if (!disabled_all_repos)
+            {
+              disable_all_repos (self);
+              disabled_all_repos = TRUE;
+            }
           if (!enable_repos (self, (const char *const*)enabled_repos, error))
             return FALSE;
         }
+
+      /* only enable lockfile-repos if we actually have a lockfile so we don't even waste
+       * time fetching metadata */
+      if (self->vlockmap)
+        {
+          g_autofree char **enabled_lockfile_repos = NULL;
+          if (g_variant_dict_lookup (self->spec->dict, "lockfile-repos", "^a&s", &enabled_lockfile_repos))
+            {
+              if (!disabled_all_repos)
+                {
+                  disable_all_repos (self);
+                  disabled_all_repos = TRUE;
+                }
+              if (!enable_repos (self, (const char *const*)enabled_lockfile_repos, error))
+                return FALSE;
+            }
+        }
     }
 
   g_autoptr(GPtrArray) repos =
@@ -2048,6 +2078,21 @@ rpmostree_context_prepare (RpmOstreeContext *self,
         }
       else
         {
+          /* Exclude all the packages in lockfile repos except locked packages. */
+          g_autofree char **lockfile_repos = NULL;
+          g_variant_dict_lookup (self->spec->dict, "lockfile-repos", "^a&s", &lockfile_repos);
+          for (char **it = lockfile_repos; it && *it; it++)
+            {
+              const char *repo = *it;
+              hy_autoquery HyQuery query = hy_query_create (sack);
+              hy_query_filter (query, HY_PKG_REPONAME, HY_EQ, repo);
+              DnfPackageSet *pset = hy_query_run_set (query);
+              Map *map = dnf_packageset_get_map (pset);
+              map_subtract (map, dnf_packageset_get_map (locked_pset));
+              dnf_sack_add_excludes (sack, pset);
+              dnf_packageset_free (pset);
+            }
+
           /* In relaxed mode, we allow packages to be added or removed without having to
            * edit lockfiles. However, we still want to make sure that if a package does get
            * installed which is in the lockfile, it can only pick that NEVRA. To do this, we

tests/compose.sh

@@ -126,6 +126,7 @@ EOF
 import sys, json
 y = json.load(sys.stdin)
 y["repos"] = ["cache"]
+y.pop("lockfile-repos", None)
 json.dump(y, sys.stdout)' < manifest.json > manifest.json.new
   mv manifest.json{.new,}
   git add .

tests/compose/test-lockfile.sh

@@ -151,3 +151,37 @@ if runcompose \
 fi
 assert_file_has_content err.txt "Couldn't find locked package 'unmatched-pkg-1.0-1.x86_64'"
 echo "ok strict mode locked pkg missing from rpmmd"
+
+# test lockfile-repos, i.e. check that a pkg in a lockfile repo with higher
+# NEVRA isn't picked unless if it's not in the lockfile
+
+# some file shuffling to get a separate yumrepo-locked/ which has foobar-2.0
+build_rpm foobar
+mv yumrepo yumrepo.bak
+build_rpm foobar version 2.0
+mv yumrepo yumrepo-locked
+mv yumrepo.bak yumrepo
+sed -e 's/test-repo/test-lockfile-repo/g' < yumrepo.repo > yumrepo-locked.repo
+sed -e 's/yumrepo/yumrepo-locked/g' < yumrepo-locked.repo > yumrepo-locked.repo.new
+mv yumrepo-locked.repo.new yumrepo-locked.repo
+ln "$PWD/yumrepo-locked.repo" config/yumrepo-locked.repo
+treefile_append "packages" '["foobar"]'
+
+# try first as a regular repo, to make sure it's functional
+treefile_append "repos" '["test-lockfile-repo"]'
+runcompose \
+  --ex-lockfile="$PWD/versions.lock" \
+  --ex-write-lockfile-to="$PWD/versions.lock.new" \
+  --dry-run "${treefile}" |& tee out.txt
+assert_file_has_content out.txt 'foobar-2.0-1.x86_64'
+
+# ok, now as a lockfile repo
+treefile_remove "repos" '"test-lockfile-repo"'
+treefile_append "lockfile-repos" '["test-lockfile-repo"]'
+runcompose \
+  --ex-lockfile="$PWD/versions.lock" \
+  --ex-write-lockfile-to="$PWD/versions.lock.new" \
+  --dry-run "${treefile}" |& tee out.txt
+assert_file_has_content out.txt 'foobar-1.0-1.x86_64'
+treefile_remove "packages" '"foobar"'
+echo "ok lockfile-repos"