ci: Rework installed tests to use Fedora Standard Test interface

Reusing the way `standard-test-roles` has support for booting
a qcow2 actually gets us to the "VM-in-container" flow.  Plus
Ansible over shell script is sometimes nicer.

https://fedoraproject.org/wiki/CI/Tests#Testing_an_Atomic_Host

It's better than what we were doing before for installed tests,
and moreover using Ansible more broadly for testing is going
to align us better with Fedora's CI.

As part of this I split off a "libpaprci" which I intend to maintain
as a "copylib" for a little bit between ostree/rpm-ostree, and then
we'll figure out how to expand from there (maybe some of the patterns
get "baked in" to PAPR for example).

Note the `FAH27-insttests` context moves to the top since it's now
of primary importance, and I expect that we start expanding it.

Closes: #1462
Approved by: jlebon

.papr.yml

@@ -1,54 +1,45 @@
+# https://fedoraproject.org/wiki/CI/Tests
 branches:
     - master
     - auto
     - try
 
+context: FAH27-insttests
 required: true
-context: f27-primary
 
 container:
-    image: registry.fedoraproject.org/fedora:27
+  image: registry.fedoraproject.org/fedora:27
 
+tests:
+  - cd tests/fedora-str && ../../ci/build-rpm.sh
+  - ./tests/fedora-str/provision.sh
+  # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
+  - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
+  - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/run.sh
+
+artifacts:
+  - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log
+  - tests/fedora-str/artifacts/installed-tests.log
+
+---
+
+# This suite skips the RPMs and does the build+unit tests in a container
+inherit: true
+context: f27-primary
 env:
-    # Enable all the sanitizers for this primary build.
+  # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain
-    # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain
+  CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2'
-    CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2'
+  # Only for CI with a known g-ir-scanner
-    # Only for CI with a known g-ir-scanner
+  GI_SCANNERFLAGS: '--warn-error'
-    GI_SCANNERFLAGS: '--warn-error'
+  ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
-    ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
+  # TODO when we're doing leak checks: G_SLICE: "always-malloc"
-    # TODO when we're doing leak checks: G_SLICE: "always-malloc"
+  CONFIGOPTS: '--with-curl --with-openssl'
-    CONFIGOPTS: '--with-curl --with-openssl'
 
 tests:
   - ci/ci-commitmessage-submodules.sh
   - ci/build-check.sh
   - ci/ci-release-build.sh
 
-timeout: 30m
-
-# Keep this in sync with build-check.sh
-artifacts:
-  - test-suite.log
-  - config.log
-  - gdtr-results
----
-
-context: c7-primary
-inherit: true
-required: true
-
-host:
-  distro: centos/7/atomic
-
-env:
-    CFLAGS: ''
-    CONFIGOPTS: '--with-curl --with-openssl'
-
-tests:
-  - docker run --privileged -v $PWD:$PWD --workdir $PWD
-    registry.centos.org/centos/centos:7 sh -c
-    'yum install -y git && ci/build-check.sh'
-
 ---
 
 context: f27-rust
@@ -132,33 +123,6 @@ tests:
 
 ---
 
-inherit: false
-branches:
-    - master
-    - auto
-    - try
-
-context: f27ah-insttest
-required: false
-
-cluster:
-  hosts:
-    - name: vmcheck
-      distro: fedora/27/atomic
-  container:
-    image: registry.fedoraproject.org/fedora:27
-
-# Copy the build from the container to the host; ideally down the line
-# this is installing an RPM via https://github.com/jlebon/redhat-ci/issues/10
-tests:
-  - ci/build.sh
-  - make install DESTDIR=$(pwd)/insttree
-  - yum -y install rsync
-  - rsync -rl -e 'ssh -o User=root' . vmcheck:ostree/
-  - ssh root@vmcheck './ostree/tests/installed/fah-prep.sh && ./ostree/tests/installed/run.sh'
-
----
-
 inherit: false
 branches:
     - master

ci/build-check.sh

@@ -4,7 +4,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 ${dn}/build.sh
 topdir=$(git rev-parse --show-toplevel)
 resultsdir=$(mktemp -d)
@@ -14,9 +14,21 @@ make syntax-check  # TODO: do syntax-check under check
 for x in test-suite.log config.log; do
     mv ${x} ${resultsdir}
 done
-# And now run the installed tests
+# And now install; we'll run the test suite after we do a clang build first
+# (But we don't install that one)
 make install
 
+# And now a clang build to find unused variables because it does a better
+# job than gcc for vars with cleanups; perhaps in the future these could
+# parallelize
+if test -x /usr/bin/clang; then
+    # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument]
+    export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}"
+    git clean -dfx && git submodule foreach git clean -dfx
+    export CC=clang
+    build
+fi
+
 copy_out_gdtr_artifacts() {
     # Keep this in sync with papr.yml
     # TODO; Split the main/clang builds into separate build dirs
@@ -40,14 +52,3 @@ if test -x /usr/bin/gnome-desktop-testing-runner; then
     # Use the new -L option
     gnome-desktop-testing-runner -L ${resultsdir}/gdtr-results -p 0 ${INSTALLED_TESTS_PATTERN:-libostree/}
 fi
-
-# And now a clang build to find unused variables because it does a better
-# job than gcc for vars with cleanups; perhaps in the future these could
-# parallelize
-if test -x /usr/bin/clang; then
-    # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument]
-    export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}"
-    git clean -dfx && git submodule foreach git clean -dfx
-    export CC=clang
-    build
-fi

ci/build-rpm.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/bash
+# Generate a src.rpm, then binary rpms in the current directory
+
+set -xeuo pipefail
+
+dn=$(dirname $0)
+paprcidir=${dn}/libpaprci
+. ${paprcidir}/libbuild.sh
+
+# Auto-provision bootstrap resources if run as root (normally in CI)
+if test "$(id -u)" == 0; then
+    pkg_install_buildroot
+    pkg_install make /usr/bin/rpmbuild git
+fi
+
+# PAPR really should do this
+if ! test -f libglnx/README.md || ! test -f bsdiff/README.md; then
+    git submodule update --init
+fi
+
+# Default libcurl on by default in fedora unless libsoup is enabled
+if test "${OS_ID}" = 'fedora'; then
+    case "${CONFIGOPTS:-}" in
+        *--with-soup*|*--without-curl*) ;;
+        *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl"
+    esac
+fi
+case "${CONFIGOPTS:-}" in
+    *--with-curl*|*--with-soup*)
+        if test -x /usr/bin/gnome-desktop-testing-runner; then
+            CONFIGOPTS="${CONFIGOPTS} --enable-installed-tests=exclusive"
+        fi
+        ;;
+esac
+
+# TODO: Use some form of rpm's --build-in-place to skip archive-then-unpack?
+make -f ${paprcidir}/Makefile.dist-packaging srpm PACKAGE=libostree DISTGIT_NAME=ostree
+if test "$(id -u)" == 0; then
+    pkg_builddep *.src.rpm
+else
+    echo "NOTE: Running as non-root, assuming build dependencies are installed"
+fi
+if ! ${paprcidir}/rpmbuild-cwd --rebuild *.src.rpm; then
+    find . -type f -name config.log -exec cat {} \;
+    exit 1
+fi

ci/build.sh

@@ -4,12 +4,11 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 pkg_upgrade
-pkg_install_builddeps ostree
+pkg_install_buildroot
-# Until this propagates farther
+pkg_builddep ostree
-pkg_install 'pkgconfig(libcurl)' 'pkgconfig(openssl)'
 pkg_install sudo which attr fuse strace \
     libubsan libasan libtsan PyYAML redhat-rpm-config \
     elfutils
@@ -18,15 +17,13 @@ if test -n "${CI_PKGS:-}"; then
 fi
 pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang \
                   python3-PyYAML
-(. /etc/os-release;
+if test "${OS_ID}" = "centos"; then
- if test "${ID}" = "centos"; then
+    rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    pkg_install python34{,-PyYAML}
-     pkg_install python34{,-PyYAML}
+fi
- fi
-)
 
 # Default libcurl on by default in fedora unless libsoup is enabled
-if sh -c '. /etc/os-release; test "${ID}" = fedora'; then
+if test "${OS_ID}" = 'fedora'; then
     case "${CONFIGOPTS:-}" in
         *--with-soup*|*--without-curl*) ;;
         *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl"

ci/ci-commitmessage-submodules.sh

@@ -17,7 +17,7 @@ set -euo pipefail
 # being tested rather than the merge sha
 HEAD=${PAPR_COMMIT:-HEAD}
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 tmpd=$(mktemp -d)
 touch ${tmpd}/.tmpdir

ci/flatpak.sh

@@ -9,13 +9,13 @@ set -xeuo pipefail
 FLATPAK_TAG=0.10.2.1
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 codedir=$(pwd)
 
 pkg_upgrade
-pkg_install_builddeps ostree
+pkg_install_buildroot
-pkg_install_builddeps flatpak
+pkg_builddep ostree flatpak
 pkg_install gettext-devel # A new dependency
 # Copy of builddeps from build.sh in flatpak
 pkg_install sudo which attr fuse \

ci/libbuild.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/bash
-
-pkg_upgrade() {
-    # https://bugzilla.redhat.com/show_bug.cgi?id=1483553
-    ecode=0
-    yum -y distro-sync 2>err.txt || ecode=$?
-    if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
-        rpm --rebuilddb
-        yum -y distro-sync
-    else
-        if test ${ecode} '!=' 0; then
-            cat err.txt
-            exit ${ecode}
-        fi
-    fi
-}
-
-make() {
-    /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@"
-}
-
-build() {
-    env NOCONFIGURE=1 ./autogen.sh
-    ./configure --prefix=/usr --libdir=/usr/lib64 "$@"
-    make V=1
-}
-
-pkg_install() {
-    yum -y install "$@"
-}
-
-pkg_install_if_os() {
-    os=$1
-    shift
-    (. /etc/os-release;
-         if test "${os}" = "${ID}"; then
-            pkg_install "$@"
-         else
-             echo "Skipping installation on OS ${ID}: $@"
-         fi
-    )
-}
-
-pkg_builddep() {
-    # This is sadly the only case where it's a different command
-    if test -x /usr/bin/dnf; then
-        dnf builddep -y "$@"
-    else
-        yum-builddep -y "$@"
-    fi
-}
-
-pkg_install_builddeps() {
-    pkg=$1
-    if test -x /usr/bin/dnf; then
-        yum -y install dnf-plugins-core
-        yum install -y 'dnf-command(builddep)'
-        # Base buildroot
-        pkg_install @buildsys-build
-    else
-        yum -y install yum-utils
-        # Base buildroot, copied from the mock config sadly
-        yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz
-    fi
-    # builddeps+runtime deps
-    pkg_builddep $pkg
-    pkg_install $pkg
-    rpm -e $pkg
-}

ci/libpaprci/Makefile.dist-packaging

@@ -0,0 +1,39 @@
+# -*- mode: Makefile -*-
+
+mypath = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+topsrcdir = $(shell git rev-parse --show-toplevel)
+GITREV = $(shell git describe --always --tags)
+GITREV_FOR_PKG = $(shell echo "$(GITREV)" | sed -e 's,-,\.,g' -e 's,^v,,')
+
+PACKAGE ?= $(shell basename $(topsrcdir))
+DISTGIT_NAME ?= $(PACKAGE)
+DISTGIT ?= https://src.fedoraproject.org/rpms/$(DISTGIT_NAME)
+SPEC ?= $(topsrcdir)/$(DISTGIT_NAME).spec
+
+PKG_VER = $(PACKAGE)-$(GITREV_FOR_PKG)
+PKG_CLIENT_VER = $(PACKAGE)-client-$(GITREV_FOR_PKG)
+
+dist-snapshot:
+	if ! test -f $(PKG_VER).tar.xz; then \
+	  $(mypath)/make-git-snapshot.sh "$(topsrcdir)" "$(PKG_VER)" "$(GITREV)" && \
+	  rm -f $(PKG_VER).tar.xz && \
+	  xz $(PKG_VER).tar; \
+  fi
+
+srpm: dist-snapshot
+	if test -f "$(SPEC)"; then \
+	   sed -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(SPEC) > $(DISTGIT_NAME).spec && \
+	   $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec ; \
+	else \
+	  test -d $(DISTGIT_NAME) || git clone --depth=1 $(DISTGIT) && \
+    mv $(PKG_VER).tar.xz $(DISTGIT_NAME) && \
+    origdir=$$(pwd); \
+	  cd $(DISTGIT_NAME) && \
+		git stash && git pull -r && \
+	  sed -i -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(DISTGIT_NAME).spec && \
+    rm -f *.src.rpm && \
+	  $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec && mv *.src.rpm $${origdir}; \
+	fi
+
+rpm: srpm
+	./rpmbuild-cwd --rebuild $(PKG_VER)*.src.rpm

ci/libpaprci/libbuild.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/bash
+
+dn=$(cd $(dirname $0) && pwd)
+
+OS_ID=$(. /etc/os-release; echo $ID)
+OS_VERSION_ID=$(. /etc/os-release; echo $VERSION_ID)
+
+pkg_upgrade() {
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1483553
+    local ecode=0
+    yum -y distro-sync 2>err.txt || ecode=$?
+    if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
+        rpm --rebuilddb
+        yum -y distro-sync
+    else
+        if test ${ecode} '!=' 0; then
+            cat err.txt
+            exit ${ecode}
+        fi
+    fi
+}
+
+make() {
+    /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@"
+}
+
+build() {
+    env NOCONFIGURE=1 ./autogen.sh
+    ./configure --sysconfdir=/etc --prefix=/usr --libdir=/usr/lib64 "$@"
+    make V=1
+}
+
+pkg_install() {
+    yum -y install "$@"
+}
+
+pkg_install_if_os() {
+    local os=$1
+    shift
+    if test "${os}" = "${OS_ID}"; then
+        pkg_install "$@"
+    else
+        echo "Skipping installation targeted for ${os} (current OS: ${OS_ID}): $@"
+    fi
+}
+
+pkg_install_buildroot() {
+    case "${OS_ID}" in
+        fedora) pkg_install dnf-plugins-core @buildsys-build;;
+        centos) pkg_install yum-utils
+                # Base buildroot, copied from the mock config sadly
+                pkg_install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ \
+                            grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar \
+                            unzip util-linux which xz;;
+        *) fatal "pkg_install_buildroot(): Unhandled OS ${OS_ID}";;
+    esac
+}
+
+pkg_builddep() {
+    # This is sadly the only case where it's a different command
+    if test -x /usr/bin/dnf; then
+        dnf builddep -y "$@"
+    else
+        yum-builddep -y "$@"
+    fi
+}
+
+# Install both build and runtime dependencies for $pkg
+pkg_builddep_runtimedep() {
+    local pkg=$1
+    pkg_builddep $pkg
+    pkg_install $pkg
+    rpm -e $pkg
+}

ci/libpaprci/make-git-snapshot.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+srcdir=$1
+shift
+PKG_VER=$1
+shift
+GITREV=$1
+shift
+
+TARFILE=${PKG_VER}.tar
+TARFILE_TMP=${TARFILE}.tmp
+
+set -x
+set -e
+
+test -n "${srcdir}"
+test -n "${PKG_VER}"
+test -n "${GITREV}"
+
+TOP=$(git rev-parse --show-toplevel)
+
+echo "Archiving ${PKG_VER} at ${GITREV} to ${TARFILE_TMP}"
+(cd ${TOP}; git archive --format=tar --prefix=${PKG_VER}/ ${GITREV}) > ${TARFILE_TMP}
+ls -al ${TARFILE_TMP}
+(cd ${TOP}; git submodule status) | while read line; do
+    rev=$(echo ${line} | cut -f 1 -d ' '); path=$(echo ${line} | cut -f 2 -d ' ')
+    echo "Archiving ${path} at ${rev}"
+    (cd ${srcdir}/${path}; git archive --format=tar --prefix=${PKG_VER}/${path}/ ${rev}) > submodule.tar
+    tar -A -f ${TARFILE_TMP} submodule.tar
+    rm submodule.tar
+done
+mv ${TARFILE_TMP} ${TARFILE}

ci/libpaprci/rpmbuild-cwd

@@ -0,0 +1,11 @@
+#!/bin/sh
+# rpmbuild-cwd:
+# Run "rpmbuild", defining all RPM variables to use the current directory.
+# This matches Fedora's system.
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+pwd=$(pwd)
+exec rpmbuild --define "_sourcedir ${pwd}" --define "_specdir ${pwd}" --define "_builddir ${pwd}" --define "_srcrpmdir ${pwd}" --define "_rpmdir ${pwd}" --define "_buildrootdir ${pwd}/.build" "$@"

ci/provision-prep.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/bash
+# Prepare the current environment
+
+set -xeuo pipefail
+
+dn=$(dirname $0)
+. ${dn}/libpaprci/libbuild.sh
+pkg_upgrade
+pkg_install_buildroot
+pkg_install sudo which attr fuse strace \
+            libubsan libasan libtsan PyYAML elfutils
+pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang
+
+if test -n "${CI_PKGS:-}"; then
+    pkg_install ${CI_PKGS}
+fi

ci/rpmostree.sh

@@ -9,13 +9,13 @@ set -xeuo pipefail
 RPMOSTREE_TAG=v2017.11
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 codedir=$(pwd)
 
 pkg_upgrade
-pkg_install_builddeps ostree
+pkg_install_buildroot
-pkg_install_builddeps rpm-ostree
+pkg_builddep ostree rpm-ostree
 pkg_install rpm-ostree && rpm -e rpm-ostree
 
 # Duplicate of deps from build.sh in rpm-ostree for tests

tests/fedora-str/README.md

@@ -0,0 +1,2 @@
+This directory holds tests that use the
+[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface).

tests/fedora-str/overlay-git.yml

@@ -0,0 +1,31 @@
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  tasks:
+  - command: ostree --version
+    changed_when: False
+    register: ostree_orig_version
+  - set_fact:
+      ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}"
+  - name: Copy locally built RPMs
+    synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes
+  - shell: ostree admin unlock || true
+  # Install the RPMs we already have.  For the test suite we use rpm2cpio
+  # since it depends on libsoup, but we're not using that yet for the sysinstalled tests
+  - shell: >
+      /usr/bin/rpm -Fvh /root/x86_64/*.rpm && \
+      cd / && rpm2cpio /root/x86_64/ostree-tests-2*.rpm | cpio -div
+  - command: ostree --version
+    register: ostree_new_version
+  - set_fact:
+      ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}"
+  - name: "Fail if we didn't change the ostree version"
+    when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git']
+    fail:
+      msg: "Failed to change ostree version"
+
+  # Next copy all of the tests/ directory
+  - name: Copy test data
+    synchronize: src=../../ dest=/root/tests/ archive=yes

tests/fedora-str/provision.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+set -xeuo pipefail
+
+dn=$(dirname $0)
+. ${dn}/../../ci/libpaprci/libbuild.sh
+
+pkg_upgrade
+pkg_install git rsync openssh-clients ansible standard-test-roles

tests/fedora-str/run.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/bash
+set -xeuo pipefail
+
+dn=$(cd $(dirname $0) && pwd)
+cd ${dn}
+
+# https://fedoraproject.org/wiki/CI/Tests
+if test -z "${TEST_SUBJECTS:-}"; then
+    cat <<EOF
+
+error: TEST_SUBJECTS must be set; e.g.:
+
+  curl -Lo fedora-atomic-host.qcow2 'https://getfedora.org/atomic_qcow2_latest'
+  export TEST_SUBJECTS=\$(pwd)/fedora-atomic-host.qcow2
+
+If you're doing interactive development, we recommend caching the qcow2
+somewhere persistent.
+EOF
+    exit 1
+fi
+ls -al ${TEST_SUBJECTS}
+
+export ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY:-$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory)}
+ls -al /dev/kvm
+ansible-playbook --tags=atomic tests.yml

tests/fedora-str/sysinstall-tests.yml

@@ -0,0 +1,20 @@
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  tasks:
+  # Down the line perhaps do each log file separately?
+  - name: Run sysinstalled tests
+    shell: /root/tests/installed/run.sh &> /root/installed-tests.log
+    register: sysinstalled_result
+    failed_when: False
+  - name: Fetch sysinstalled results
+    fetch:
+      src: /root/installed-tests.log
+      dest: artifacts/installed-tests.log
+      flat: yes
+  - name: Assert that sysinstalled tests succeeded
+    when: sysinstalled_result.rc != 0
+    fail:
+      msg: "sysinstalled tests failed"

tests/fedora-str/tests.yml

@@ -0,0 +1,2 @@
+- include: overlay-git.yml
+- include: sysinstall-tests.yml

tests/installed/fah-prep.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -xeuo pipefail
-# If we're using devmapper, expand the root
-if lvm lvs atomicos/docker-pool &>/dev/null; then
-    systemctl stop docker
-    lvm lvremove -f atomicos/docker-pool
-fi
-lvm lvextend -r -l +100%FREE atomicos/root
-ostree admin unlock
-rsync -rlv ./ostree/insttree/usr/ /usr/