extensions: Add support for development extensions

In RHCOS, we ship kernel development-related packages as an extension.
Those aren't really extensions that are meant to be layered onto the
host.  They're meant to be used in a build environment somewhere to
compile kernel modules.

This makes it very different from "OS extensions" in at least two
drastic ways:
1. we don't want to do any depsolving (e.g. we don't want to pull in
   `gcc` or something)
2. some of those packages may be present in the base already, but we
   still want to redownload them

Hesitated putting this functionality in rpm-ostree, but I think in the
end it cuts from the benefit of moving this code to rpm-ostree if we
can't entirely get rid of the Python script it obsoletes. Plus, being
able to use the `match-base-evr` is still really useful for this use
case.

Let's add a new `kind` key to support this. The traditional extensions
are called "OS extensions" and these new extensions are called
"development extensions".

The latter is not yet part of the state checksum, so change detection
doesn't work there. I think that's fine for now though because the
primary use case is the kernel, and there we want to match the base
version. So if the kernel changes, the base would change too. (Though
there's the corner case of adding a new package to the list while at the
same version...)

docs/extensions.md

@@ -36,6 +36,9 @@ extensions:
   # This can be whatever name you'd like. The name itself
   # isn't used by rpm-ostree.
   sooper-dooper-tracers:
+    # Optional; defaults to `os-extension`. An OS extension
+    # is an extension intended to be `rpm-ostree install`ed.
+    kind: os-extension
     # List of packages for this extension
     packages:
         - strace
@@ -47,6 +50,11 @@ extensions:
         - x86_64
         - aarch64
   kernel-dev:
+    # A development extension lists packages useful for
+    # developing for the target OSTree, but won't be layered
+    # on top. A common example is kernel modules. No
+    # depsolving happens, packages listed are downloaded.
+    kind: development
     packages:
       - kernel-devel
       - kernel-headers

rust/src/extensions.rs

@@ -33,6 +33,21 @@ pub struct Extension {
     architectures: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     match_base_evr: Option<String>,
+    #[serde(default)]
+    kind: ExtensionKind,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+enum ExtensionKind {
+    OsExtension,
+    Development,
+}
+
+impl Default for ExtensionKind {
+    fn default() -> Self {
+        ExtensionKind::OsExtension
+    }
 }
 
 fn extensions_load_stream(
@@ -55,9 +70,11 @@ fn extensions_load_stream(
         .collect();
 
     for (_, ext) in parsed.extensions.iter_mut() {
-        for pkg in &ext.packages {
+        if ext.kind == ExtensionKind::OsExtension {
-            if base_pkgs.contains_key(pkg.as_str()) {
+            for pkg in &ext.packages {
-                bail!("package {} already present in base", pkg);
+                if base_pkgs.contains_key(pkg.as_str()) {
+                    bail!("package {} already present in base", pkg);
+                }
             }
         }
         if let Some(ref matched_base_pkg) = ext.match_base_evr {
@@ -92,9 +109,18 @@ impl Extensions {
         self.repos.as_ref().map(|v| v.clone()).unwrap_or_default()
     }
 
-    pub(crate) fn get_packages(&self) -> Vec<String> {
+    pub(crate) fn get_os_extension_packages(&self) -> Vec<String> {
         self.extensions
             .iter()
+            .filter(|(_, ext)| ext.kind == ExtensionKind::OsExtension)
+            .flat_map(|(_, ext)| ext.packages.iter().cloned())
+            .collect()
+    }
+
+    pub(crate) fn get_development_packages(&self) -> Vec<String> {
+        self.extensions
+            .iter()
+            .filter(|(_, ext)| ext.kind == ExtensionKind::Development)
             .flat_map(|(_, ext)| ext.packages.iter().cloned())
             .collect()
     }
@@ -157,7 +183,8 @@ extensions:
         let mut input = std::io::BufReader::new(buf.as_bytes());
         let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
         assert!(extensions.get_repos() == vec!["my-repo"]);
-        assert!(extensions.get_packages() == vec!["bazboo"]);
+        assert!(extensions.get_os_extension_packages() == vec!["bazboo"]);
+        assert!(extensions.get_development_packages().is_empty());
     }
 
     #[test]
@@ -175,6 +202,21 @@ extensions:
         }
     }
 
+    #[test]
+    fn ext_in_devel() {
+        let buf = r###"
+extensions:
+    foobar:
+        packages:
+            - foobar
+        kind: development
+"###;
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
+        assert!(extensions.get_os_extension_packages().is_empty());
+        assert!(extensions.get_development_packages() == vec!["foobar"]);
+    }
+
     #[test]
     fn basearch_filter() {
         let buf = r###"
@@ -193,10 +235,10 @@ extensions:
 "###;
         let mut input = std::io::BufReader::new(buf.as_bytes());
         let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
-        assert!(extensions.get_packages() == vec!["bazboo"]);
+        assert!(extensions.get_os_extension_packages() == vec!["bazboo"]);
         let mut input = std::io::BufReader::new(buf.as_bytes());
         let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap();
-        assert!(extensions.get_packages() == vec!["dodo", "dada"]);
+        assert!(extensions.get_os_extension_packages() == vec!["dodo", "dada"]);
     }
 
     #[test]
@@ -207,9 +249,16 @@ extensions:
         packages:
             - foobar-ext
         match-base-evr: foobar
+        kind: os-extension
+    devel:
+        packages:
+            - foobar-devel
+        match-base-evr: foobar
+        kind: development
 "###;
         let mut input = std::io::BufReader::new(buf.as_bytes());
         let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
-        assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]);
+        assert!(extensions.get_os_extension_packages() == vec!["foobar-ext-1.2-3"]);
+        assert!(extensions.get_development_packages() == vec!["foobar-devel-1.2-3"]);
     }
 }

rust/src/lib.rs

@@ -191,7 +191,8 @@ pub mod ffi {
             base_pkgs: &Vec<StringMapping>,
         ) -> Result<Box<Extensions>>;
         fn get_repos(&self) -> Vec<String>;
-        fn get_packages(&self) -> Vec<String>;
+        fn get_os_extension_packages(&self) -> Vec<String>;
+        fn get_development_packages(&self) -> Vec<String>;
         fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result<bool>;
         fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>;
         fn serialize_to_dir(&self, output_dir: &str) -> Result<()>;

src/app/rpmostree-compose-builtin-tree.cxx

@@ -1561,7 +1561,7 @@ rpmostree_compose_builtin_extensions (int             argc,
 
   g_autoptr(RpmOstreeTreespec) spec = NULL;
   { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free);
-    auto pkgs = extensions->get_packages();
+    auto pkgs = extensions->get_os_extension_packages();
     for (auto pkg : pkgs)
       g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str()));
 
@@ -1622,6 +1622,44 @@ rpmostree_compose_builtin_extensions (int             argc,
         return FALSE;
     }
 
+  /* This is hacky: for "development" extensions, we don't want any depsolving
+   * against the base OS. Rather than awkwardly teach the core about this, we
+   * just reuse its sack and keep all the functionality here. */
+
+  DnfContext *dnfctx = rpmostree_context_get_dnf (ctx);
+  DnfSack *sack = dnf_context_get_sack (dnfctx);
+
+  /* disable the system repo; we always want to download, even if already in the base */
+  dnf_sack_repo_enabled (sack, HY_SYSTEM_REPO_NAME, 0);
+
+  auto pkgs = extensions->get_development_packages();
+  g_autoptr(GPtrArray) devel_pkgs_to_download =
+    g_ptr_array_new_with_free_func (g_object_unref);
+  for (auto pkg : pkgs)
+    {
+      g_autoptr(GPtrArray) matches = rpmostree_get_matching_packages (sack, pkg.c_str());
+      if (matches->len == 0)
+        return glnx_throw (error, "Package %s not found", pkg.c_str());
+      DnfPackage *found_pkg = (DnfPackage*)matches->pdata[0];
+      g_ptr_array_add (devel_pkgs_to_download, g_object_ref (found_pkg));
+    }
+
+  rpmostree_set_repos_on_packages (dnfctx, devel_pkgs_to_download);
+
+  if (!rpmostree_download_packages (devel_pkgs_to_download, cancellable, error))
+    return FALSE;
+
+  for (guint i = 0; i < devel_pkgs_to_download->len; i++)
+    {
+      DnfPackage *pkg = (DnfPackage*)devel_pkgs_to_download->pdata[i];
+      const char *src = dnf_package_get_filename (pkg);
+      const char *basename = glnx_basename (src);
+      if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd,
+                              basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error))
+        return FALSE;
+    }
+
+  // XXX: account for development extensions
   extensions->update_state_checksum (state_checksum, opt_extensions_output_dir);
   extensions->serialize_to_dir (opt_extensions_output_dir);
   if (!process_touch_if_changed (error))

tests/compose/test-basic-unified.sh

@@ -89,6 +89,16 @@ build_rpm dodo-base
 build_rpm dodo requires dodo-base
 build_rpm solitaire
 
+# this is pretty terrible... need --json for `rpm-ostree db list`
+kernel_vra=$(rpm-ostree db list --repo=${repo} ${treeref} kernel | tail -n1 | cut -d- -f2-)
+kernel_v=$(cut -d- -f1 <<< "$kernel_vra")
+kernel_ra=$(cut -d- -f2- <<< "$kernel_vra")
+kernel_r=${kernel_ra%.x86_64}
+
+build_rpm kernel-core version ${kernel_v} release ${kernel_r}
+build_rpm kernel-devel version ${kernel_v} release ${kernel_r}
+build_rpm kernel-headers version ${kernel_v} release ${kernel_r}
+
 cat > extensions.yaml << EOF
 extensions:
   extinct-birds:
@@ -100,6 +110,13 @@ extensions:
       - nonexistent
     architectures:
       - badarch
+  kernel-devel:
+    kind: development
+    packages:
+      - kernel-core
+      - kernel-devel
+      - kernel-headers
+    match-base-evr: kernel
 EOF
 
 # we don't actually need root here, but in CI the cache may be in a qcow2 and
@@ -110,10 +127,12 @@ runasroot rpm-ostree compose extensions --repo=${repo} \
   --touch-if-changed extensions-changed
 
 ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm
+ls extensions/kernel-{core,devel,headers}-${kernel_v}-${kernel_r}.x86_64.rpm
 test -f extensions-changed
 assert_jq extensions/extensions.json \
-  '.extensions|length == 1' \
+  '.extensions|length == 2' \
-  '.extensions["extinct-birds"]'
+  '.extensions["extinct-birds"]' \
+  '.extensions["kernel-devel"]'
 echo "ok extensions"
 
 rm extensions-changed