Add new Rust-based tests

There's a lot going on here.  First, this is intended to run
nicely as part of the new [cosa/kola ext-tests](https://github.com/coreos/coreos-assembler/pull/1252).

With Rust we can get one big static binary that we can upload,
and include a webserver as part of the binary.  This way we don't
need to do the hack of running a container with Python or whatever.

Now, what's even better about Rust for this is that it has macros,
and specifically we are using [commandspec](https://github.com/tcr/commandspec/)
which allows us to "inline" shell script.  I think the macros
could be even better, but this shows how we can intermix
pure Rust code along with using shell safely enough.

We're using my fork of commandspec because the upstream hasn't
merged [a few PRs](https://github.com/tcr/commandspec/pulls?q=is%3Apr+author%3Acgwalters+).

This model is intended to replace *both* some of our
`make check` tests as well.

Oh, and this takes the obvious step of using the Rust OSTree bindings
as part of our tests.  Currently the "commandspec tests" and "API tests"
are separate, but nothing stops us from intermixing them if we wanted.

I haven't yet tried to write destructive tests with this but
I think it will go well.

.cci.jenkinsfile

@@ -62,7 +62,7 @@ codestyle: {
 // Build FCOS and do a kola basic run
 stage("More builds and test") {
 parallel fcos: {
-  cosaPod(runAsUser: 0, memory: "2048Mi", cpu: "2") {
+  cosaPod(buildroot: true, runAsUser: 0, memory: "3072Mi", cpu: "4") {
     stage("Build FCOS") {
       checkout scm
       unstash 'build'

tests/basic-test.sh

@@ -21,7 +21,7 @@
 
 set -euo pipefail
 
-echo "1..$((89 + ${extra_basic_tests:-0}))"
+echo "1..$((86 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -1031,17 +1031,6 @@ stat '--format=%Y' test2-checkout/baz/deeper > deeper-mtime
 assert_file_has_content deeper-mtime 0
 echo "ok content mtime"
 
-cd ${test_tmpdir}
-rm -rf test2-checkout
-mkdir -p test2-checkout
-cd test2-checkout
-mkfifo afifo
-if $OSTREE commit ${COMMIT_ARGS} -b test2 -s "Attempt to commit a FIFO" 2>../errmsg; then
-    assert_not_reached "Committing a FIFO unexpetedly succeeded!"
-    assert_file_has_content ../errmsg "Unsupported file type"
-fi
-echo "ok commit of fifo was rejected"
-
 cd ${test_tmpdir}
 rm repo2 -rf
 mkdir repo2
@@ -1180,22 +1169,3 @@ if test "$(id -u)" != "0"; then
 else
     echo "ok # SKIP not run when root"
 fi
-
-cd ${test_tmpdir}
-rm -rf test2-checkout
-mkdir -p test2-checkout
-cd test2-checkout
-touch blah
-stat --printf="%.Y\n" ${test_tmpdir}/repo > ${test_tmpdir}/timestamp-orig.txt
-$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Should bump the mtime"
-stat --printf="%.Y\n" ${test_tmpdir}/repo > ${test_tmpdir}/timestamp-new.txt
-cd ..
-if cmp timestamp-{orig,new}.txt; then
-    assert_not_reached "failed to update mtime on repo"
-fi
-echo "ok mtime updated"
-
-cd ${test_tmpdir}
-$OSTREE init --mode=bare --repo=repo-extensions
-assert_has_dir repo-extensions/extensions
-echo "ok extensions dir"

tests/inst/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

tests/inst/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "ostree-test"
+version = "0.1.0"
+authors = ["Colin Walters <walters@verbum.org>"]
+edition = "2018"
+
+[[bin]]
+name = "ostree-test"
+path = "src/insttest.rs"
+
+[dependencies]
+clap = "2.32.0"
+structopt = "0.2"
+commandspec = "0.12.2"
+anyhow = "1.0"
+tempfile = "3.1.0"
+gio = "0.8"
+ostree = { version = "0.7.1", features = ["v2020_1"] }
+libtest-mimic = "0.2.0"
+twoway = "0.2.1"
+hyper = "0.13"
+futures = "0.3.4"
+http = "0.2.0"
+hyper-staticfile = "0.5.1"
+tokio = { version = "0.2", features = ["full"] }
+futures-util = "0.3.1"
+base64 = "0.12.0"
+procspawn = "0.8"
+proc-macro2 = "0.4"
+quote = "0.6"
+syn = "0.15"
+linkme = "0.2"
+
+itest-macro = { path = "itest-macro" }
+
+with-procspawn-tempdir = { git = "https://github.com/cgwalters/with-procspawn-tempdir" }
+#with-procspawn-tempdir = { path = "/var/srv/walters/src/github/cgwalters/with-procspawn-tempdir" }
+
+# See https://github.com/tcr/commandspec/pulls?q=is%3Apr+author%3Acgwalters+
+[patch.crates-io]
+commandspec = { git = "https://github.com/cgwalters/commandspec", branch = 'walters-master' }
+#commandspec = { path = "/var/srv/walters/src/github/tcr/commandspec" }

tests/inst/itest-macro/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "itest-macro"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+proc-macro = true
+path = "src/itest-macro.rs"
+
+[dependencies]
+quote = "1.0.3"
+proc-macro2 = "1.0.10"
+syn = { version = "1.0.3", features = ["full"] }
+anyhow = "1.0"

tests/inst/itest-macro/src/itest-macro.rs

@@ -0,0 +1,32 @@
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+
+/// Wraps function using `procspawn` to allocate a new temporary directory,
+/// make it the process' working directory, and run the function.
+#[proc_macro_attribute]
+pub fn itest(attrs: TokenStream, input: TokenStream) -> TokenStream {
+    let attrs = syn::parse_macro_input!(attrs as syn::AttributeArgs);
+    if attrs.len() > 0 {
+        return syn::Error::new_spanned(&attrs[0], "itest takes no attributes")
+            .to_compile_error()
+            .into();
+    }
+    let func = syn::parse_macro_input!(input as syn::ItemFn);
+    let fident = func.sig.ident.clone();
+    let varident = quote::format_ident!("ITEST_{}", fident);
+    let fidentstrbuf = format!(r#"{}"#, fident);
+    let fidentstr = syn::LitStr::new(&fidentstrbuf, Span::call_site());
+    let output = quote! {
+        #[linkme::distributed_slice(TESTS)]
+        #[allow(non_upper_case_globals)]
+        static #varident : Test = Test {
+            name: #fidentstr,
+            f: #fident,
+        };
+        #func
+    };
+    output.into()
+}

tests/inst/src/insttest.rs

@@ -0,0 +1,46 @@
+use anyhow::Result;
+// use structopt::StructOpt;
+// // https://github.com/clap-rs/clap/pull/1397
+// #[macro_use]
+// extern crate clap;
+
+mod repobin;
+mod sysroot;
+mod test;
+
+fn gather_tests() -> Vec<test::TestImpl> {
+    test::TESTS
+        .iter()
+        .map(|t| libtest_mimic::Test {
+            name: t.name.into(),
+            kind: "".into(),
+            is_ignored: false,
+            is_bench: false,
+            data: t,
+        })
+        .collect()
+}
+
+fn run_test(test: &test::TestImpl) -> libtest_mimic::Outcome {
+    if let Err(e) = (test.data.f)() {
+        libtest_mimic::Outcome::Failed {
+            msg: Some(e.to_string()),
+        }
+    } else {
+        libtest_mimic::Outcome::Passed
+    }
+}
+
+fn main() -> Result<()> {
+    procspawn::init();
+
+    // Ensure we're always in tempdir so we can rely on it globally
+    let tmp_dir = tempfile::Builder::new()
+        .prefix("ostree-insttest-top")
+        .tempdir()?;
+    std::env::set_current_dir(tmp_dir.path())?;
+
+    let args = libtest_mimic::Arguments::from_args();
+    let tests = gather_tests();
+    libtest_mimic::run_tests(&args, tests, run_test).exit();
+}

tests/inst/src/repobin.rs

@@ -0,0 +1,121 @@
+//! Tests that mostly use the CLI and operate on temporary
+//! repositories.
+
+use std::path::Path;
+
+use crate::test::*;
+use anyhow::{Context, Result};
+use commandspec::{sh_command, sh_execute};
+use tokio::runtime::Runtime;
+use with_procspawn_tempdir::with_procspawn_tempdir;
+
+#[itest]
+fn test_basic() -> Result<()> {
+    sh_execute!(r"ostree --help >/dev/null")?;
+    Ok(())
+}
+
+#[itest]
+#[with_procspawn_tempdir]
+fn test_nofifo() -> Result<()> {
+    assert!(std::path::Path::new(".procspawn-tmpdir").exists());
+    sh_execute!(
+        r"ostree --repo=repo init --mode=archive
+    mkdir tmproot
+    mkfifo tmproot/afile
+"
+    )?;
+    cmd_fails_with(
+        sh_command!(
+            r#"ostree --repo=repo commit -b fifotest -s "commit fifo" --tree=dir=./tmproot"#
+        )
+        .unwrap(),
+        "Not a regular file or symlink",
+    )?;
+    Ok(())
+}
+
+#[itest]
+#[with_procspawn_tempdir]
+fn test_mtime() -> Result<()> {
+    sh_execute!(
+        r"ostree --repo=repo init --mode=archive
+    mkdir tmproot
+    echo afile > tmproot/afile
+    ostree --repo=repo commit -b test --tree=dir=tmproot >/dev/null
+"
+    )?;
+    let ts = Path::new("repo").metadata()?.modified().unwrap();
+    sh_execute!(
+        r#"ostree --repo=repo commit -b test -s "bump mtime" --tree=dir=tmproot >/dev/null"#
+    )?;
+    assert_ne!(ts, Path::new("repo").metadata()?.modified().unwrap());
+    Ok(())
+}
+
+#[itest]
+#[with_procspawn_tempdir]
+fn test_extensions() -> Result<()> {
+    sh_execute!(r"ostree --repo=repo init --mode=bare")?;
+    assert!(Path::new("repo/extensions").exists());
+    Ok(())
+}
+
+async fn impl_test_pull_basicauth() -> Result<()> {
+    let opts = TestHttpServerOpts {
+        basicauth: true,
+        ..Default::default()
+    };
+    let serverrepo = Path::new("server/repo");
+    std::fs::create_dir_all(&serverrepo)?;
+    let addr = http_server(&serverrepo, opts).await?;
+    tokio::task::spawn_blocking(move || -> Result<()> {
+        let baseuri = http::Uri::from_maybe_shared(format!("http://{}/", addr).into_bytes())?;
+        let unauthuri =
+            http::Uri::from_maybe_shared(format!("http://unknown:badpw@{}/", addr).into_bytes())?;
+        let authuri = http::Uri::from_maybe_shared(
+            format!("http://{}@{}/", TEST_HTTP_BASIC_AUTH, addr).into_bytes(),
+        )?;
+        let osroot = Path::new("osroot");
+        mkroot(&osroot)?;
+        sh_execute!(
+            r#"ostree --repo={serverrepo} init --mode=archive
+        ostree --repo={serverrepo} commit -b os --tree=dir={osroot} >/dev/null
+        mkdir client
+        cd client
+        ostree --repo=repo init --mode=archive
+        ostree --repo=repo remote add --set=gpg-verify=false origin-unauth {baseuri}
+        ostree --repo=repo remote add --set=gpg-verify=false origin-badauth {unauthuri}
+        ostree --repo=repo remote add --set=gpg-verify=false origin-goodauth {authuri}
+        "#,
+            osroot = osroot.to_str(),
+            serverrepo = serverrepo.to_str(),
+            baseuri = baseuri.to_string(),
+            unauthuri = unauthuri.to_string(),
+            authuri = authuri.to_string()
+        )?;
+        for rem in &["unauth", "badauth"] {
+            cmd_fails_with(
+                sh_command!(
+                    r#"ostree --repo=client/repo pull origin-{rem} os >/dev/null"#,
+                    rem = *rem
+                )
+                .unwrap(),
+                "HTTP 403",
+            )
+            .context(rem)?;
+        }
+        sh_execute!(r#"ostree --repo=client/repo pull origin-goodauth os >/dev/null"#,)?;
+        Ok(())
+    })
+    .await??;
+    Ok(())
+}
+
+#[itest]
+#[with_procspawn_tempdir]
+fn test_pull_basicauth() -> Result<()> {
+    let mut rt = Runtime::new()?;
+    rt.block_on(async move { impl_test_pull_basicauth().await })?;
+    Ok(())
+}

tests/inst/src/sysroot.rs

@@ -0,0 +1,33 @@
+//! Tests that mostly use the API and access the booted sysroot read-only.
+
+use anyhow::Result;
+use gio::prelude::*;
+use ostree::prelude::*;
+
+use crate::test::*;
+
+#[itest]
+fn test_sysroot_ro() -> Result<()> {
+    // TODO add a skipped identifier
+    if !std::path::Path::new("/run/ostree-booted").exists() {
+        return Ok(());
+    }
+    let cancellable = Some(gio::Cancellable::new());
+    let sysroot = ostree::Sysroot::new_default();
+    sysroot.load(cancellable.as_ref())?;
+    assert!(sysroot.is_booted());
+
+    let booted = sysroot.get_booted_deployment().expect("booted deployment");
+    assert!(!booted.is_staged());
+    let repo = sysroot.repo().expect("repo");
+
+    let csum = booted.get_csum().expect("booted csum");
+    let csum = csum.as_str();
+
+    let (root, rev) = repo.read_commit(csum, cancellable.as_ref())?;
+    assert_eq!(rev, csum);
+    let root = root.downcast::<ostree::RepoFile>().expect("downcast");
+    root.ensure_resolved()?;
+
+    Ok(())
+}

tests/inst/src/test.rs

@@ -0,0 +1,180 @@
+use std::borrow::BorrowMut;
+use std::fs::File;
+use std::io::prelude::*;
+use std::path::Path;
+use std::process::Command;
+
+use anyhow::{bail, Context, Result};
+use linkme::distributed_slice;
+
+pub use itest_macro::itest;
+pub use with_procspawn_tempdir::with_procspawn_tempdir;
+
+// HTTP Server deps
+use futures_util::future;
+use hyper::service::{make_service_fn, service_fn};
+use hyper::{Body, Request, Response};
+use hyper_staticfile::Static;
+
+pub(crate) type TestFn = fn() -> Result<()>;
+
+#[derive(Debug)]
+pub(crate) struct Test {
+    pub(crate) name: &'static str,
+    pub(crate) f: TestFn,
+}
+
+pub(crate) type TestImpl = libtest_mimic::Test<&'static Test>;
+
+#[distributed_slice]
+pub(crate) static TESTS: [Test] = [..];
+
+/// Run command and assert that its stderr contains pat
+pub(crate) fn cmd_fails_with<C: BorrowMut<Command>>(mut c: C, pat: &str) -> Result<()> {
+    let c = c.borrow_mut();
+    let o = c.output()?;
+    if o.status.success() {
+        bail!("Command {:?} unexpectedly succeeded", c);
+    }
+    if !twoway::find_bytes(&o.stderr, pat.as_bytes()).is_some() {
+        dbg!(String::from_utf8_lossy(&o.stdout));
+        dbg!(String::from_utf8_lossy(&o.stderr));
+        bail!("Command {:?} stderr did not match: {}", c, pat);
+    }
+    Ok(())
+}
+
+pub(crate) fn write_file<P: AsRef<Path>>(p: P, buf: &str) -> Result<()> {
+    let p = p.as_ref();
+    let mut f = File::create(p)?;
+    f.write_all(buf.as_bytes())?;
+    f.flush()?;
+    Ok(())
+}
+
+pub(crate) fn mkroot<P: AsRef<Path>>(p: P) -> Result<()> {
+    let p = p.as_ref();
+    for v in &["usr/bin", "etc"] {
+        std::fs::create_dir_all(p.join(v))?;
+    }
+    let verpath = p.join("etc/version");
+    let v: u32 = if verpath.exists() {
+        let s = std::fs::read_to_string(&verpath)?;
+        let v: u32 = s.trim_end().parse()?;
+        v + 1
+    } else {
+        0
+    };
+    write_file(&verpath, &format!("{}", v))?;
+    write_file(p.join("usr/bin/somebinary"), &format!("somebinary v{}", v))?;
+    write_file(p.join("etc/someconf"), &format!("someconf v{}", v))?;
+    write_file(p.join("usr/bin/vmod2"), &format!("somebinary v{}", v % 2))?;
+    write_file(p.join("usr/bin/vmod3"), &format!("somebinary v{}", v % 3))?;
+    Ok(())
+}
+
+#[derive(Default, Debug, Copy, Clone)]
+pub(crate) struct TestHttpServerOpts {
+    pub(crate) basicauth: bool,
+}
+
+pub(crate) const TEST_HTTP_BASIC_AUTH: &'static str = "foouser:barpw";
+
+fn validate_authz(value: &[u8]) -> Result<bool> {
+    let buf = std::str::from_utf8(&value)?;
+    if let Some(o) = buf.find("Basic ") {
+        let (_, buf) = buf.split_at(o + "Basic ".len());
+        let buf = base64::decode(buf).context("decoding")?;
+        let buf = std::str::from_utf8(&buf)?;
+        Ok(buf == TEST_HTTP_BASIC_AUTH)
+    } else {
+        bail!("Missing Basic")
+    }
+}
+
+pub(crate) async fn http_server<P: AsRef<Path>>(
+    p: P,
+    opts: TestHttpServerOpts,
+) -> Result<std::net::SocketAddr> {
+    let addr = ([127, 0, 0, 1], 0).into();
+    let sv = Static::new(p.as_ref());
+
+    async fn handle_request<B: std::fmt::Debug>(
+        req: Request<B>,
+        sv: Static,
+        opts: TestHttpServerOpts,
+    ) -> Result<Response<Body>> {
+        if opts.basicauth {
+            if let Some(ref authz) = req.headers().get(http::header::AUTHORIZATION) {
+                match validate_authz(authz.as_ref()) {
+                    Ok(true) => {
+                        return Ok(sv.clone().serve(req).await?);
+                    }
+                    Ok(false) => {
+                        // Fall through
+                    }
+                    Err(e) => {
+                        return Ok(Response::builder()
+                            .status(hyper::StatusCode::INTERNAL_SERVER_ERROR)
+                            .body(Body::from(e.to_string()))
+                            .unwrap());
+                    }
+                }
+            };
+            return Ok(Response::builder()
+                .status(hyper::StatusCode::FORBIDDEN)
+                .header("x-test-auth", "true")
+                .body(Body::from("not authorized\n"))
+                .unwrap());
+        }
+        Ok(sv.clone().serve(req).await?)
+    }
+
+    let make_service = make_service_fn(move |_| {
+        let sv = sv.clone();
+        let opts = opts.clone();
+        future::ok::<_, hyper::Error>(service_fn(move |req| handle_request(req, sv.clone(), opts)))
+    });
+    let server: hyper::Server<_, _, _> = hyper::Server::bind(&addr).serve(make_service);
+    let addr = server.local_addr();
+    tokio::spawn(async move {
+        let r = server.await;
+        dbg!("server finished!");
+        r
+    });
+    Ok(addr)
+}
+
+// I put tests in your tests so you can test while you test
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn oops() -> Command {
+        let mut c = Command::new("/bin/bash");
+        c.args(&["-c", "echo oops 1>&2; exit 1"]);
+        c
+    }
+
+    #[test]
+    fn test_fails_with_matches() -> Result<()> {
+        cmd_fails_with(Command::new("false"), "")?;
+        cmd_fails_with(oops(), "oops")?;
+        Ok(())
+    }
+
+    #[test]
+    fn test_fails_with_fails() {
+        cmd_fails_with(Command::new("true"), "somepat").expect_err("true");
+        cmd_fails_with(oops(), "nomatch").expect_err("nomatch");
+    }
+
+    #[test]
+    fn test_validate_authz() -> Result<()> {
+        assert!(validate_authz("Basic Zm9vdXNlcjpiYXJwdw==".as_bytes())?);
+        assert!(!validate_authz("Basic dW5rbm93bjpiYWRwdw==".as_bytes())?);
+        assert!(validate_authz("Basic oops".as_bytes()).is_err());
+        assert!(validate_authz("oops".as_bytes()).is_err());
+        Ok(())
+    }
+}

tests/kola/nondestructive/.gitignore

@@ -0,0 +1,2 @@
+# Generated by runkola.sh
+insttest-rs

tests/kolainst/Makefile

@@ -7,7 +7,9 @@ KOLA_TESTDIR = $(DESTDIR)/usr/lib/coreos-assembler/tests/kola/ostree/
 
 all:
 	for x in $(LIBSCRIPTS); do bash -n "$${x}"; done
+	(cd ../inst && cargo build --release)
 
-install:
+install: all
 	install -D -m 0644 -t $(KOLA_TESTDIR) $(LIBSCRIPTS)
 	for x in $(TESTDIRS); do rsync -rlv ./$${x} $(KOLA_TESTDIR)/; done
+	install -D -m 0755 -t $(KOLA_TESTDIR)/nondestructive-rs ../inst/target/release/ostree-test

tests/runkola

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -euo pipefail
+# Generate a new qemu image and run tests
+top=$(git rev-parse --show-toplevel)
+cd ${top}
+make
+cosa build-fast
+image=$(ls fastbuild-*-qemu.qcow2 | head -1)
+if [ -z "${image}" ]; then 
+    echo "failed to find image"; exit 1
+fi
+if [ -z "$@" ]; then
+    set -- 'ext.ostree.*' "$@"
+fi
+set -x
+make -C tests/kolainst
+sudo make -C tests/kolainst install
+exec kola run -p qemu --qemu-image "${image}" -E ${top} "$@"