Add fedora-integration: Support override replace https://bodhi/...

This adds support for e.g.:

```
$ rpm-ostree override replace https://bodhi.fedoraproject.org/updates/FEDORA-2020-2908628031
```

This will find the Koji builds from the listed update, download
all the RPMs (that aren't debuginfo) and pass them for overrides
in the same way we support `override replace http://somewebserver/foo.rpm`
now.

We also support directly linking a Koji build:
```
$ rpm-ostree override replace https://koji.fedoraproject.org/koji/buildinfo?buildID=1625029
```

Bodhi has a modern HTTP+JSON API, and the lack of a Koji equivalent
drove me to create https://github.com/cgwalters/koji-sane-json-api
and we currently depend on an instance set up in the OpenShift CI
cluster.

I hope it shouldn't take long to deploy this in Fedora Infra,
but I don't want to block on it.

Also notably this still downloads *all* the other RPMs even
ones that aren't installed.  Handling that truly correctly
would require moving this logic to the daemon and core.

All of this functionality is keyed off a `cfg(feature = "fedora-integration")`
that is detected by a Rust `build.rs` which parses the build environment's
`/etc/os-release` for now.

Cargo.toml

@@ -40,6 +40,7 @@ libdnf-sys = { path = "rust/libdnf-sys", version = "0.1.0" }
 
 [build-dependencies]
 cbindgen = "0.16.0"
+anyhow = "1.0"
 
 [lib]
 name = "rpmostree_rust"
@@ -58,5 +59,6 @@ lto = true
 
 [features]
 sqlite-rpmdb-default = []
+fedora-integration = []
 
 default = []

build.rs

@@ -0,0 +1,21 @@
+use anyhow::Result;
+
+fn detect_fedora_feature() -> Result<()> {
+    if !std::path::Path::new("/usr/lib/os-release").exists() {
+        return Ok(());
+    }
+    let p = std::process::Command::new("sh")
+        .args(&["-c", ". /usr/lib/os-release && echo ${ID}"])
+        .stdout(std::process::Stdio::piped())
+        .output()?;
+    let out = std::str::from_utf8(&p.stdout).ok().map(|s| s.trim());
+    if out == Some("fedora") {
+        println!(r#"cargo:rustc-cfg=feature="fedora-integration""#)
+    }
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    detect_fedora_feature()?;
+    Ok(())
+}

rust/src/client.rs

@@ -10,7 +10,12 @@ fn is_http(arg: &str) -> bool {
 /// RPM URLs we need to fetch, and if so download those URLs and return file
 /// descriptors for the content.
 /// TODO(cxx-rs): This would be slightly more elegant as Result<Option<Vec<i32>>>
-pub(crate) fn client_handle_fd_argument(arg: &str, _arch: &str) -> Result<Vec<i32>> {
+pub(crate) fn client_handle_fd_argument(arg: &str, arch: &str) -> Result<Vec<i32>> {
+    #[cfg(feature = "fedora-integration")]
+    if let Some(fds) = crate::fedora_integration::handle_cli_arg(arg, arch)? {
+        return Ok(fds.into_iter().map(|f| f.into_raw_fd()).collect());
+    }
+
     if is_http(arg) {
         utils::download_url_to_tmpfile(arg, true).map(|f| vec![f.into_raw_fd()])
     } else if arg.ends_with(".rpm") {

rust/src/fedora_integration.rs

@@ -0,0 +1,175 @@
+use anyhow::{Context, Result};
+use serde_derive::Deserialize;
+use std::borrow::Cow;
+use std::fs::File;
+
+const KOJI_URL_PREFIX: &str = "https://koji.fedoraproject.org/koji/";
+/// See https://github.com/cgwalters/koji-sane-json-api
+const KOJI_JSON_API_URL: &str = "kojiproxy-coreos.svc.ci.openshift.org";
+const BODHI_URL_PREFIX: &str = "https://bodhi.fedoraproject.org/updates/";
+const BODHI_UPDATE_PREFIX: &str = "FEDORA-";
+
+mod bodhi {
+    use super::*;
+
+    #[derive(Deserialize)]
+    pub(crate) struct BodhiKojiBuild {
+        pub(crate) nvr: String,
+        /// Not currently included in koji URLs, so we ignore it
+        #[allow(dead_code)]
+        pub(crate) epoch: u64,
+    }
+
+    #[derive(Deserialize)]
+    pub(crate) struct BodhiUpdate {
+        pub(crate) builds: Vec<BodhiKojiBuild>,
+    }
+
+    #[derive(Deserialize)]
+    struct BodhiUpdateResponse {
+        update: BodhiUpdate,
+    }
+
+    pub(crate) fn canonicalize_update_id(id: &str) -> Cow<str> {
+        let id = match id.strip_prefix(BODHI_URL_PREFIX) {
+            Some(s) => return Cow::Borrowed(s),
+            None => id,
+        };
+        match id.strip_prefix(BODHI_UPDATE_PREFIX) {
+            Some(_) => Cow::Borrowed(id),
+            None => Cow::Owned(format!("{}{}", BODHI_UPDATE_PREFIX, id)),
+        }
+    }
+
+    pub(crate) fn get_update(updateid: &str) -> Result<BodhiUpdate> {
+        let updateid = canonicalize_update_id(updateid);
+        let url = format!("{}{}", BODHI_URL_PREFIX, updateid);
+        let update_data = crate::utils::download_url_to_tmpfile(&url, false)
+            .context("Failed to download bodhi update info")?;
+        let resp: BodhiUpdateResponse = serde_json::from_reader(update_data)?;
+        Ok(resp.update)
+    }
+
+    pub(crate) fn get_rpm_urls_from_update(updateid: &str, arch: &str) -> Result<Vec<String>> {
+        let update = bodhi::get_update(updateid)?;
+        update.builds.iter().try_fold(Vec::new(), |mut r, buildid| {
+            // For now hardcode skipping debuginfo because it's large and hopefully
+            // people aren't layering that.
+            let rpms = koji::get_rpm_urls_from_build(&buildid.nvr, arch, true)?;
+            r.extend(rpms);
+            Ok(r)
+        })
+    }
+
+    #[cfg(test)]
+    mod test {
+        use super::*;
+
+        #[test]
+        fn test_canonicalize() {
+            let regid = "FEDORA-2020-053d8a2e94";
+            let shortid = "2020-053d8a2e94";
+            let url = "https://bodhi.fedoraproject.org/updates/FEDORA-2020-053d8a2e94";
+            assert_eq!(canonicalize_update_id(regid), regid);
+            assert_eq!(canonicalize_update_id(url), regid);
+            assert_eq!(canonicalize_update_id(shortid), regid);
+        }
+    }
+}
+
+mod koji {
+    use super::*;
+    use std::collections::BTreeMap;
+
+    #[derive(Default, Deserialize)]
+    #[serde(rename_all = "kebab-case")]
+    #[allow(dead_code)]
+    pub(crate) struct KojiBuildInfo {
+        nvr: String,
+        id: u64,
+        kojipkgs_url_prefix: String,
+        rpms: BTreeMap<String, Vec<String>>,
+    }
+
+    impl KojiBuildInfo {
+        fn rpmurl(&self, arch: &str, rpm: &str) -> String {
+            format!("{}/{}/{}", self.kojipkgs_url_prefix, arch, rpm)
+        }
+    }
+
+    pub(crate) fn get_buildid_from_url(url: &str) -> Result<&str> {
+        let id = url.rsplit('?').next().expect("split");
+        match id.strip_prefix("buildID=") {
+            Some(s) => Ok(s),
+            None => anyhow::bail!("Failed to parse Koji buildid from URL {}", url),
+        }
+    }
+
+    pub(crate) fn get_build(buildid: &str) -> Result<KojiBuildInfo> {
+        let url = format!("{}/buildinfo/{}", KOJI_JSON_API_URL, buildid);
+        let f = crate::utils::download_url_to_tmpfile(&url, false)
+            .context("Failed to download buildinfo from koji proxy")?;
+        Ok(serde_json::from_reader(std::io::BufReader::new(f))?)
+    }
+
+    pub(crate) fn get_rpm_urls_from_build(
+        buildid: &str,
+        arch: &str,
+        skip_debug: bool,
+    ) -> Result<impl IntoIterator<Item = String>> {
+        let build = get_build(buildid)?;
+        let mut ret = Vec::new();
+        if let Some(rpms) = build.rpms.get(arch) {
+            ret.extend(
+                rpms.iter()
+                    .filter(|r| !(skip_debug && is_debug_rpm(r)))
+                    .map(|r| build.rpmurl(arch, r)),
+            );
+        }
+        if let Some(rpms) = build.rpms.get("noarch") {
+            ret.extend(rpms.iter().map(|r| build.rpmurl("noarch", r)));
+        }
+        Ok(ret.into_iter())
+    }
+
+    #[cfg(test)]
+    mod test {
+        use super::*;
+
+        #[test]
+        fn test_url_buildid() -> Result<()> {
+            assert_eq!(
+                get_buildid_from_url(
+                    "https://koji.fedoraproject.org/koji/buildinfo?buildID=1637715"
+                )?,
+                "1637715"
+            );
+            Ok(())
+        }
+    }
+}
+
+fn is_debug_rpm(rpm: &str) -> bool {
+    rpm.contains("-debuginfo-") || rpm.contains("-debugsource-")
+}
+
+pub(crate) fn handle_cli_arg(url: &str, arch: &str) -> Result<Option<Vec<File>>> {
+    if url.starts_with(BODHI_URL_PREFIX) {
+        let urls = bodhi::get_rpm_urls_from_update(url, arch)?;
+        Ok(Some(
+            crate::utils::download_urls_to_tmpfiles(urls, true)
+                .context("Failed to download RPMs")?,
+        ))
+    } else if url.starts_with(KOJI_URL_PREFIX) {
+        let buildid = koji::get_buildid_from_url(url)?;
+        let urls: Vec<String> = koji::get_rpm_urls_from_build(&buildid, arch, true)?
+            .into_iter()
+            .collect();
+        Ok(Some(
+            crate::utils::download_urls_to_tmpfiles(urls, true)
+                .context("Failed to download RPMs")?,
+        ))
+    } else {
+        Ok(None)
+    }
+}

rust/src/lib.rs

@@ -118,6 +118,8 @@ mod composepost;
 pub(crate) use composepost::*;
 mod core;
 use crate::core::*;
+#[cfg(feature = "fedora-integration")]
+mod fedora_integration;
 mod history;
 pub use self::history::*;
 mod journal;

tests/kolainst/destructive/layering-fedorainfra

@@ -0,0 +1,20 @@
+#!/bin/bash
+# kola: { "tags": "needs-internet" }
+# Test https://github.com/coreos/rpm-ostree/pull/2420
+# i.e. using overrides from Fedora Infrastructure tools (koji/bodhi)
+set -euo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+cd $(mktemp -d)
+
+# bodhi update for rpm-ostree (Fedora 33)
+rpm-ostree override replace https://bodhi.fedoraproject.org/updates/FEDORA-2020-6e743def1d
+rpm-ostree status > status.txt
+assert_file_has_content_literal status.txt "Diff: 2 downgraded"
+rpm-ostree cleanup -p
+# Same build directly via Koji
+rpm-ostree override replace https://koji.fedoraproject.org/koji/buildinfo?buildID=1637715
+rpm-ostree status > status.txt
+assert_file_has_content_literal status.txt "Diff: 2 downgraded"
+
+echo "ok"