app: Add rpm-ostree compose extensions

This adds support for a new `rpm-ostree compose extensions` command`
which takes a treefile, a new extensions YAML file, and an OSTree repo
and ref. It performs a depsolve and downloads the extensions to a
provided output directory.

This is intended to replace cosa's `download-extensions`:
https://github.com/coreos/coreos-assembler/blob/master/src/download-extensions

The input YAML schema matches the one accepted by that script.

Some differences from the script:
- We have a guaranteed depsolve match and thus can avoid silly issues
  we've hit in RHCOS (like downloading the wrong `libprotobuf` for
  `usbguard` -- rhbz#1889694).
- We seamlessly re-use the same repos defined in the treefile, whereas
  the cosa script uses `reposdir=$dir` which doesn't have the same
  semantics (repo enablement is in that case purely based on the
  `enabled` flag in those repos, which may be different than what the
  rpm-ostree compose ran with).
- We perform more sanity-checks against the requested extensions, such
  as whether the extension is already in the base.
- We support no-change detection via a state SHA512 file for better
  integration in cosa and pipelines.
- We support a `match-base-evr` key, which forces the extension to have
  the same EVR as the one from a base package: this is helpful in the
  case of extensions which complement a base package, esp. those which
  may not have strong enough reldeps to enforce matching EVRs by
  depsolve alone (`kernel-headers` is an example of this).
- We don't try to organize the RPMs into separate directories by
  extension because IMO it's not at the right level. Instead, we should
  work towards higher-level metadata to represent extensions (see
  https://github.com/openshift/os/issues/409 which is related to this).

Closes: #2055
This commit is contained in:
Jonathan Lebon 2021-01-11 16:53:05 -05:00 committed by OpenShift Merge Robot
parent 40af45814c
commit 271954a41c
14 changed files with 506 additions and 7 deletions

4
Cargo.lock generated
View File

@ -772,9 +772,9 @@ dependencies = [
[[package]]
name = "openat-ext"
version = "0.1.10"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7ac688336340b9ce22dd83e3b26d9d9063ceef5990679f75176b7e17f4e6a51"
checksum = "5157ebc7a2da568f161a0d51d355b8520451fffb66c416617236f7c8dda733be"
dependencies = [
"drop_bomb",
"libc",

View File

@ -24,7 +24,7 @@ tempfile = "3.1.0"
clap = "2.33.3"
structopt = "0.3.21"
openat = "0.1.19"
openat-ext = "^0.1.9"
openat-ext = "^0.1.11"
curl = "0.4.34"
rayon = "1.5.0"
c_utf8 = "0.1.0"

View File

@ -1,5 +1,5 @@
---
nav_order: 9
nav_order: 10
---
# Contributing

View File

@ -1,5 +1,5 @@
---
nav_order: 6
nav_order: 7
---
# Hacking on rpm-ostree

View File

@ -1,5 +1,5 @@
---
nav_order: 7
nav_order: 8
---
# Releasing rpm-ostree

56
docs/extensions.md Normal file
View File

@ -0,0 +1,56 @@
---
nav_order: 6
---
# Extensions
Extensions are additional packages which client machines can
install using package layering. While rpm-ostree itself is
indifferent on the subject, most rpm-ostree-based distros
encourage a containerized workflow for better separation of
host and application layers. But sometimes, containerization
is not ideal for some software, and yet it may not be
desirable to bake them into the OSTree commit by default.
Package layering normally fetches such extensions from
remote repos. However in some architectures there may be a
better way to transfer them, or one may simply want tighter
control over them and stronger binding between OSTree commit
and extension versions (e.g. for reproducibility, guaranteed
depsolve, QE, releng, etc..).
`rpm-ostree compose extensions` takes an `extensions.yaml`
file describing OS extensions (packages) and a base OSTree
commit. After performing a depsolve, it downloads the
extension packages and places them in an output directory.
## extensions.yaml
The format of the `extensions.yaml` file is as follow:
```yaml
# The top-level object is a dict. The only supported key
# right now is `extensions`, which is a dict of extension
# names to extension objects.
extensions:
# This can be whatever name you'd like. The name itself
# isn't used by rpm-ostree.
sooper-dooper-tracers:
# List of packages for this extension
packages:
- strace
- ltrace
# Optional list of architectures on which this extension
# is valid. These are RPM basearches. If omitted,
# defaults to all architectures.
architectures:
- x86_64
- aarch64
kernel-dev:
packages:
- kernel-devel
- kernel-headers
# Optional name of a base package used to constrain the
# EVR of all the packages in this extension.
match-base-evr: kernel
```

View File

@ -1,5 +1,5 @@
---
nav_order: 8
nav_order: 9
---
# Repository structure

194
rust/src/extensions.rs Normal file
View File

@ -0,0 +1,194 @@
//! Core logic for extensions.yaml file.
/*
* Copyright (C) 2020 Red Hat, Inc.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
use anyhow::{bail, Context, Result};
use openat_ext::OpenatDirExt;
use serde_derive::Deserialize;
use std::collections::HashMap;
use crate::cxxrsutil::*;
use crate::ffi::StringMapping;
use crate::utils;
const RPMOSTREE_EXTENSIONS_STATE_FILE: &str = ".rpm-ostree-state-chksum";
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
pub struct Extensions {
extensions: HashMap<String, Extension>,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "kebab-case")]
pub struct Extension {
packages: Vec<String>,
architectures: Option<Vec<String>>,
match_base_evr: Option<String>,
}
fn extensions_load_stream(
stream: &mut impl std::io::Read,
basearch: &str,
base_pkgs: &Vec<StringMapping>,
) -> Result<Box<Extensions>> {
let mut parsed: Extensions = serde_yaml::from_reader(stream)?;
parsed.extensions.retain(|_, ext| {
ext.architectures
.as_ref()
.map(|v| v.iter().any(|a| a == basearch))
.unwrap_or(true)
});
let base_pkgs: HashMap<&str, &str> = base_pkgs
.iter()
.map(|i| (i.k.as_str(), i.v.as_str()))
.collect();
for (_, ext) in parsed.extensions.iter_mut() {
for pkg in &ext.packages {
if base_pkgs.contains_key(pkg.as_str()) {
bail!("package {} already present in base", pkg);
}
}
if let Some(ref matched_base_pkg) = ext.match_base_evr {
let evr = base_pkgs
.get(matched_base_pkg.as_str())
.with_context(|| format!("couldn't find base package {}", matched_base_pkg))?;
let pkgs = ext
.packages
.iter()
.map(|pkg| format!("{}-{}", pkg, evr))
.collect();
ext.packages = pkgs;
}
}
Ok(Box::new(parsed))
}
pub(crate) fn extensions_load(
path: &str,
basearch: &str,
base_pkgs: &Vec<StringMapping>,
) -> Result<Box<Extensions>> {
let f = utils::open_file(path)?;
let mut f = std::io::BufReader::new(f);
extensions_load_stream(&mut f, basearch, base_pkgs).with_context(|| format!("parsing {}", path))
}
impl Extensions {
pub(crate) fn get_packages(&self) -> Vec<String> {
self.extensions
.iter()
.flat_map(|(_, ext)| ext.packages.iter().cloned())
.collect()
}
pub(crate) fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> CxxResult<bool> {
let output_dir = openat::Dir::open(output_dir)?;
if let Some(prev_chksum) =
output_dir.read_to_string_optional(RPMOSTREE_EXTENSIONS_STATE_FILE)?
{
Ok(prev_chksum != chksum)
} else {
Ok(true)
}
}
pub(crate) fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> CxxResult<()> {
let output_dir = openat::Dir::open(output_dir)?;
Ok(output_dir
.write_file_contents(RPMOSTREE_EXTENSIONS_STATE_FILE, 0o644, chksum)
.with_context(|| format!("updating state file {}", RPMOSTREE_EXTENSIONS_STATE_FILE))?)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_rpmdb() -> Vec<StringMapping> {
vec![
StringMapping {
k: "systemd".into(),
v: "246.9-3".into(),
},
StringMapping {
k: "foobar".into(),
v: "1.2-3".into(),
},
]
}
#[test]
fn basic() {
let buf = r###"
extensions:
bazboo:
packages:
- bazboo
"###;
let mut input = std::io::BufReader::new(buf.as_bytes());
let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
assert!(extensions.get_packages() == vec!["bazboo"]);
}
#[test]
fn ext_in_base() {
let buf = r###"
extensions:
foobar:
packages:
- foobar
"###;
let mut input = std::io::BufReader::new(buf.as_bytes());
match extensions_load_stream(&mut input, "x86_64", &base_rpmdb()) {
Ok(_) => panic!("expected failure from extension in base"),
Err(ref e) => assert!(e.to_string() == "package foobar already present in base"),
}
}
#[test]
fn basearch_filter() {
let buf = r###"
extensions:
bazboo:
packages:
- bazboo
architectures:
- x86_64
dodo:
packages:
- dodo
- dada
architectures:
- s390x
"###;
let mut input = std::io::BufReader::new(buf.as_bytes());
let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
assert!(extensions.get_packages() == vec!["bazboo"]);
let mut input = std::io::BufReader::new(buf.as_bytes());
let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap();
assert!(extensions.get_packages() == vec!["dodo", "dada"]);
}
#[test]
fn matching_evr() {
let buf = r###"
extensions:
foobar-ext:
packages:
- foobar-ext
match-base-evr: foobar
"###;
let mut input = std::io::BufReader::new(buf.as_bytes());
let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]);
}
}

View File

@ -188,6 +188,19 @@ mod ffi {
extern "Rust" {
fn countme_entrypoint(argv: Vec<String>) -> Result<()>;
}
// extensions.rs
extern "Rust" {
type Extensions;
fn extensions_load(
path: &str,
basearch: &str,
base_pkgs: &Vec<StringMapping>,
) -> Result<Box<Extensions>>;
fn get_packages(&self) -> Vec<String>;
fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result<bool>;
fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>;
}
}
mod client;
@ -201,6 +214,8 @@ pub(crate) use composepost::*;
mod core;
use crate::core::*;
mod dirdiff;
mod extensions;
pub(crate) use extensions::*;
#[cfg(feature = "fedora-integration")]
mod fedora_integration;
mod history;

View File

@ -1477,6 +1477,16 @@ mod ffi {
ref_from_raw_ptr(tf).serialized.as_ptr()
}
#[no_mangle]
pub extern "C" fn ror_treefile_get_repos(tf: *mut Treefile) -> *mut *mut libc::c_char {
let tf = ref_from_raw_ptr(tf);
if let Some(ref repos) = tf.parsed.repos {
repos.to_glib_full()
} else {
ptr::null_mut()
}
}
#[no_mangle]
pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char {
let tf = ref_from_raw_ptr(tf);

View File

@ -41,6 +41,9 @@ static RpmOstreeCommand compose_subcommands[] = {
{ "commit", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT),
"Commit a target path to an OSTree repository",
rpmostree_compose_builtin_commit },
{ "extensions", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD,
"Download RPM packages guaranteed to depsolve with a base OSTree",
rpmostree_compose_builtin_extensions },
#ifdef BUILDOPT_ROJIG
{ "rojig", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
"EXPERIMENTAL: Build a rojig RPM from a treefile, output to a local rpm-md repo",

View File

@ -83,6 +83,9 @@ static char **opt_lockfiles;
static gboolean opt_lockfile_strict;
static char *opt_parent;
static char *opt_extensions_output_dir;
static char *opt_extensions_base_rev;
/* shared by both install & commit */
static GOptionEntry common_option_entries[] = {
{ "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
@ -124,6 +127,14 @@ static GOptionEntry commit_option_entries[] = {
{ NULL }
};
static GOptionEntry extensions_option_entries[] = {
{ "output-dir", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_output_dir, "Path to extensions output directory", "PATH" },
{ "base-rev", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_base_rev, "Base OSTree revision", "REV" },
{ "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
{ "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if new extensions were downloaded", "FILE" },
{ NULL }
};
typedef struct {
RpmOstreeContext *corectx;
GFile *treefile_path;
@ -1430,3 +1441,179 @@ rpmostree_compose_builtin_tree (int argc,
return TRUE;
}
gboolean
rpmostree_compose_builtin_extensions (int argc,
char **argv,
RpmOstreeCommandInvocation *invocation,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE EXTYAML");
g_option_context_add_main_entries (context, common_option_entries, NULL);
g_option_context_add_main_entries (context, extensions_option_entries, NULL);
if (!rpmostree_option_context_parse (context,
NULL,
&argc, &argv,
invocation,
cancellable,
NULL, NULL, NULL, NULL, NULL,
error))
return FALSE;
if (argc < 3)
{
rpmostree_usage_error (context, "TREEFILE and EXTYAML must be specified", error);
return FALSE;
}
if (!opt_repo)
{
rpmostree_usage_error (context, "--repo must be specified", error);
return FALSE;
}
if (!opt_extensions_output_dir)
{
rpmostree_usage_error (context, "--output-dir must be specified", error);
return FALSE;
}
const char *treefile_path = argv[1];
const char *extensions_path = argv[2];
g_autofree char *basearch = rpm_ostree_get_basearch ();
g_autoptr(RORTreefile) treefile = ror_treefile_new (treefile_path, basearch, -1, error);
if (!treefile)
return glnx_prefix_error (error, "Failed to load treefile");
g_autoptr(OstreeRepo) repo = ostree_repo_open_at (AT_FDCWD, opt_repo, cancellable, error);
if (!repo)
return FALSE;
/* this is a similar construction to what's in rpm_ostree_compose_context_new() */
g_auto(GLnxTmpDir) cachedir_tmp = { 0, };
glnx_autofd int cachedir_dfd = -1;
if (opt_cachedir)
{
if (!glnx_opendirat (AT_FDCWD, opt_cachedir, TRUE, &cachedir_dfd, error))
return glnx_prefix_error (error, "Opening cachedir");
}
else
{
if (!glnx_mkdtempat (ostree_repo_get_dfd (repo),
"tmp/rpm-ostree-compose.XXXXXX", 0700,
&cachedir_tmp, error))
return FALSE;
cachedir_dfd = fcntl (cachedir_tmp.fd, F_DUPFD_CLOEXEC, 3);
if (cachedir_dfd < 0)
return glnx_throw_errno_prefix (error, "fcntl");
}
g_autofree char *base_rev = NULL;
if (!ostree_repo_resolve_rev (repo, opt_extensions_base_rev, FALSE, &base_rev, error))
return FALSE;
g_autoptr(GVariant) commit = NULL;
if (!ostree_repo_load_commit (repo, base_rev, &commit, NULL, error))
return FALSE;
g_autoptr(GPtrArray) packages =
rpm_ostree_db_query_all (repo, opt_extensions_base_rev, cancellable, error);
if (!packages)
return FALSE;
auto packages_mapping = std::make_unique<rust::Vec<rpmostreecxx::StringMapping>>();
for (guint i = 0; i < packages->len; i++)
{
RpmOstreePackage *pkg = (RpmOstreePackage*)packages->pdata[i];
const char *name = rpm_ostree_package_get_name (pkg);
const char *evr = rpm_ostree_package_get_evr (pkg);
packages_mapping->push_back(rpmostreecxx::StringMapping {k: name, v: evr});
}
auto extensions = rpmostreecxx::extensions_load (extensions_path, basearch, *packages_mapping);
g_autoptr(RpmOstreeContext) ctx =
rpmostree_context_new_tree (cachedir_dfd, repo, cancellable, error);
if (!ctx)
return FALSE;
{ int tf_dfd = ror_treefile_get_dfd (treefile);
g_autofree char *abs_tf_path = glnx_fdrel_abspath (tf_dfd, ".");
dnf_context_set_repo_dir (rpmostree_context_get_dnf (ctx), abs_tf_path);
}
#define TMP_EXTENSIONS_ROOTFS "rpmostree-extensions.tmp"
if (!glnx_shutil_rm_rf_at (cachedir_dfd, TMP_EXTENSIONS_ROOTFS, cancellable, error))
return FALSE;
g_print ("Checking out %.7s... ", base_rev);
OstreeRepoCheckoutAtOptions opts = { .mode = OSTREE_REPO_CHECKOUT_MODE_USER };
if (!ostree_repo_checkout_at (repo, &opts, cachedir_dfd, TMP_EXTENSIONS_ROOTFS,
base_rev, cancellable, error))
return FALSE;
g_print ("done!\n");
g_autoptr(RpmOstreeTreespec) spec = NULL;
{ g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free);
auto pkgs = extensions->get_packages();
for (auto pkg : pkgs)
g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str()));
char **repos = ror_treefile_get_repos (treefile);
g_autoptr(GKeyFile) treespec = g_key_file_new ();
g_key_file_set_string_list (treespec, "tree", "packages",
(const char* const*)gpkgs->pdata, gpkgs->len);
g_key_file_set_string_list (treespec, "tree", "repos",
(const char* const*)repos,
g_strv_length (repos));
spec = rpmostree_treespec_new_from_keyfile (treespec, NULL);
}
g_autofree char *checkout_path = glnx_fdrel_abspath (cachedir_dfd, TMP_EXTENSIONS_ROOTFS);
if (!rpmostree_context_setup (ctx, checkout_path, checkout_path, spec, cancellable, error))
return FALSE;
#undef TMP_EXTENSIONS_ROOTFS
if (!rpmostree_context_prepare (ctx, cancellable, error))
return FALSE;
if (!glnx_shutil_mkdir_p_at (AT_FDCWD, opt_extensions_output_dir, 0755, cancellable, error))
return FALSE;
glnx_autofd int output_dfd = -1;
if (!glnx_opendirat (AT_FDCWD, opt_extensions_output_dir, TRUE, &output_dfd, error))
return glnx_prefix_error (error, "Opening output dir");
g_autofree char *state_checksum;
if (!rpmostree_context_get_state_sha512 (ctx, &state_checksum, error))
return FALSE;
if (!extensions->state_checksum_changed (state_checksum, opt_extensions_output_dir))
{
g_print ("No change.\n");
return TRUE;
}
if (!rpmostree_context_download (ctx, cancellable, error))
return FALSE;
g_autoptr(GPtrArray) extensions_pkgs = rpmostree_context_get_packages (ctx);
for (guint i = 0; i < extensions_pkgs->len; i++)
{
DnfPackage *pkg = (DnfPackage*)extensions_pkgs->pdata[i];
const char *src = dnf_package_get_filename (pkg);
const char *basename = glnx_basename (src);
if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd,
basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error))
return FALSE;
}
extensions->update_state_checksum (state_checksum, opt_extensions_output_dir);
if (!process_touch_if_changed (error))
return FALSE;
return TRUE;
}

View File

@ -31,6 +31,7 @@ gboolean rpmostree_compose_builtin_rojig (int argc, char **argv, RpmOstreeComman
gboolean rpmostree_compose_builtin_install (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_postprocess (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_commit (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
gboolean rpmostree_compose_builtin_extensions (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
G_END_DECLS

View File

@ -84,3 +84,36 @@ if ostree rev-parse --repo "${repo}" "${newrev}"^ 2>error.txt; then
fi
assert_file_has_content_literal error.txt 'has no parent'
echo "ok --no-parent"
build_rpm dodo-base
build_rpm dodo requires dodo-base
build_rpm solitaire
cat > extensions.yaml << EOF
extensions:
extinct-birds:
packages:
- dodo
- solitaire
EOF
# we don't actually need root here, but in CI the cache may be in a qcow2 and
# the supermin code is gated behind `runasroot`
runasroot rpm-ostree compose extensions --repo=${repo} \
--cachedir=${test_tmpdir}/cache --base-rev ${treeref} \
--output-dir extensions ${treefile} extensions.yaml \
--touch-if-changed extensions-changed
ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm
test -f extensions-changed
echo "ok extensions"
rm extensions-changed
runasroot rpm-ostree compose extensions --repo=${repo} \
--cachedir=${test_tmpdir}/cache --base-rev ${treeref} \
--output-dir extensions ${treefile} extensions.yaml \
--touch-if-changed extensions-changed
if test -f extensions-changed; then
fatal "found extensions-changed"
fi
echo "ok extensions no change"