From 317b92066738b43a7b35ccc81696bca7d223139e Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:19:33 -0500 Subject: [PATCH] extensions: Add support for development extensions In RHCOS, we ship kernel development-related packages as an extension. Those aren't really extensions that are meant to be layered onto the host. They're meant to be used in a build environment somewhere to compile kernel modules. This makes it very different from "OS extensions" in at least two drastic ways: 1. we don't want to do any depsolving (e.g. we don't want to pull in `gcc` or something) 2. some of those packages may be present in the base already, but we still want to redownload them Hesitated putting this functionality in rpm-ostree, but I think in the end it cuts from the benefit of moving this code to rpm-ostree if we can't entirely get rid of the Python script it obsoletes. Plus, being able to use the `match-base-evr` is still really useful for this use case. Let's add a new `kind` key to support this. The traditional extensions are called "OS extensions" and these new extensions are called "development extensions". The latter is not yet part of the state checksum, so change detection doesn't work there. I think that's fine for now though because the primary use case is the kernel, and there we want to match the base version. So if the kernel changes, the base would change too. (Though there's the corner case of adding a new package to the list while at the same version...) --- docs/extensions.md | 8 +++ rust/src/extensions.rs | 65 +++++++++++++++++++--- rust/src/lib.rs | 3 +- src/app/rpmostree-compose-builtin-tree.cxx | 40 ++++++++++++- tests/compose/test-basic-unified.sh | 23 +++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index b8960f03..8a48f1e1 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -36,6 +36,9 @@ extensions: # This can be whatever name you'd like. The name itself # isn't used by rpm-ostree. sooper-dooper-tracers: + # Optional; defaults to `os-extension`. An OS extension + # is an extension intended to be `rpm-ostree install`ed. + kind: os-extension # List of packages for this extension packages: - strace @@ -47,6 +50,11 @@ extensions: - x86_64 - aarch64 kernel-dev: + # A development extension lists packages useful for + # developing for the target OSTree, but won't be layered + # on top. A common example is kernel modules. No + # depsolving happens, packages listed are downloaded. + kind: development packages: - kernel-devel - kernel-headers diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index de4c8866..0df07457 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -33,6 +33,21 @@ pub struct Extension { architectures: Option>, #[serde(skip_serializing_if = "Option::is_none")] match_base_evr: Option, + #[serde(default)] + kind: ExtensionKind, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum ExtensionKind { + OsExtension, + Development, +} + +impl Default for ExtensionKind { + fn default() -> Self { + ExtensionKind::OsExtension + } } fn extensions_load_stream( @@ -55,9 +70,11 @@ fn extensions_load_stream( .collect(); for (_, ext) in parsed.extensions.iter_mut() { - for pkg in &ext.packages { - if base_pkgs.contains_key(pkg.as_str()) { - bail!("package {} already present in base", pkg); + if ext.kind == ExtensionKind::OsExtension { + for pkg in &ext.packages { + if base_pkgs.contains_key(pkg.as_str()) { + bail!("package {} already present in base", pkg); + } } } if let Some(ref matched_base_pkg) = ext.match_base_evr { @@ -92,9 +109,18 @@ impl Extensions { self.repos.as_ref().map(|v| v.clone()).unwrap_or_default() } - pub(crate) fn get_packages(&self) -> Vec { + pub(crate) fn get_os_extension_packages(&self) -> Vec { self.extensions .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::OsExtension) + .flat_map(|(_, ext)| ext.packages.iter().cloned()) + .collect() + } + + pub(crate) fn get_development_packages(&self) -> Vec { + self.extensions + .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::Development) .flat_map(|(_, ext)| ext.packages.iter().cloned()) .collect() } @@ -157,7 +183,8 @@ extensions: let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); assert!(extensions.get_repos() == vec!["my-repo"]); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); + assert!(extensions.get_development_packages().is_empty()); } #[test] @@ -175,6 +202,21 @@ extensions: } } + #[test] + fn ext_in_devel() { + let buf = r###" +extensions: + foobar: + packages: + - foobar + kind: development +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_os_extension_packages().is_empty()); + assert!(extensions.get_development_packages() == vec!["foobar"]); + } + #[test] fn basearch_filter() { let buf = r###" @@ -193,10 +235,10 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["dodo", "dada"]); + assert!(extensions.get_os_extension_packages() == vec!["dodo", "dada"]); } #[test] @@ -207,9 +249,16 @@ extensions: packages: - foobar-ext match-base-evr: foobar + kind: os-extension + devel: + packages: + - foobar-devel + match-base-evr: foobar + kind: development "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_os_extension_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_development_packages() == vec!["foobar-devel-1.2-3"]); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7abc38dd..65d1047d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -191,7 +191,8 @@ pub mod ffi { base_pkgs: &Vec, ) -> Result>; fn get_repos(&self) -> Vec; - fn get_packages(&self) -> Vec; + fn get_os_extension_packages(&self) -> Vec; + fn get_development_packages(&self) -> Vec; fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; fn serialize_to_dir(&self, output_dir: &str) -> Result<()>; diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 978ef3ba..ab6c27e7 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1561,7 +1561,7 @@ rpmostree_compose_builtin_extensions (int argc, g_autoptr(RpmOstreeTreespec) spec = NULL; { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free); - auto pkgs = extensions->get_packages(); + auto pkgs = extensions->get_os_extension_packages(); for (auto pkg : pkgs) g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); @@ -1622,6 +1622,44 @@ rpmostree_compose_builtin_extensions (int argc, return FALSE; } + /* This is hacky: for "development" extensions, we don't want any depsolving + * against the base OS. Rather than awkwardly teach the core about this, we + * just reuse its sack and keep all the functionality here. */ + + DnfContext *dnfctx = rpmostree_context_get_dnf (ctx); + DnfSack *sack = dnf_context_get_sack (dnfctx); + + /* disable the system repo; we always want to download, even if already in the base */ + dnf_sack_repo_enabled (sack, HY_SYSTEM_REPO_NAME, 0); + + auto pkgs = extensions->get_development_packages(); + g_autoptr(GPtrArray) devel_pkgs_to_download = + g_ptr_array_new_with_free_func (g_object_unref); + for (auto pkg : pkgs) + { + g_autoptr(GPtrArray) matches = rpmostree_get_matching_packages (sack, pkg.c_str()); + if (matches->len == 0) + return glnx_throw (error, "Package %s not found", pkg.c_str()); + DnfPackage *found_pkg = (DnfPackage*)matches->pdata[0]; + g_ptr_array_add (devel_pkgs_to_download, g_object_ref (found_pkg)); + } + + rpmostree_set_repos_on_packages (dnfctx, devel_pkgs_to_download); + + if (!rpmostree_download_packages (devel_pkgs_to_download, cancellable, error)) + return FALSE; + + for (guint i = 0; i < devel_pkgs_to_download->len; i++) + { + DnfPackage *pkg = (DnfPackage*)devel_pkgs_to_download->pdata[i]; + const char *src = dnf_package_get_filename (pkg); + const char *basename = glnx_basename (src); + if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd, + basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + return FALSE; + } + + // XXX: account for development extensions extensions->update_state_checksum (state_checksum, opt_extensions_output_dir); extensions->serialize_to_dir (opt_extensions_output_dir); if (!process_touch_if_changed (error)) diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 0578eb23..9b511c18 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -89,6 +89,16 @@ build_rpm dodo-base build_rpm dodo requires dodo-base build_rpm solitaire +# this is pretty terrible... need --json for `rpm-ostree db list` +kernel_vra=$(rpm-ostree db list --repo=${repo} ${treeref} kernel | tail -n1 | cut -d- -f2-) +kernel_v=$(cut -d- -f1 <<< "$kernel_vra") +kernel_ra=$(cut -d- -f2- <<< "$kernel_vra") +kernel_r=${kernel_ra%.x86_64} + +build_rpm kernel-core version ${kernel_v} release ${kernel_r} +build_rpm kernel-devel version ${kernel_v} release ${kernel_r} +build_rpm kernel-headers version ${kernel_v} release ${kernel_r} + cat > extensions.yaml << EOF extensions: extinct-birds: @@ -100,6 +110,13 @@ extensions: - nonexistent architectures: - badarch + kernel-devel: + kind: development + packages: + - kernel-core + - kernel-devel + - kernel-headers + match-base-evr: kernel EOF # we don't actually need root here, but in CI the cache may be in a qcow2 and @@ -110,10 +127,12 @@ runasroot rpm-ostree compose extensions --repo=${repo} \ --touch-if-changed extensions-changed ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm +ls extensions/kernel-{core,devel,headers}-${kernel_v}-${kernel_r}.x86_64.rpm test -f extensions-changed assert_jq extensions/extensions.json \ - '.extensions|length == 1' \ - '.extensions["extinct-birds"]' + '.extensions|length == 2' \ + '.extensions["extinct-birds"]' \ + '.extensions["kernel-devel"]' echo "ok extensions" rm extensions-changed