vmcheck: add a basic test harness

This is a very simple test harness inspired by the atomic one. It's a
simple bash script that sets up a permanent ssh connection to the host
and runs the test scripts. Also add a "demo" test-basic.sh test to make
sure that it works.

Closes: #344
Approved by: cgwalters

Makefile-tests.am

@@ -70,18 +70,26 @@ check-local:
 	@echo "  *** NOTE ***"
 
 vmbuild:
-	vagrant up # provision if not already done
-	vagrant rsync
-	vagrant ssh -c "cd sync/tests/vmcheck && make install VERSION=$$(git describe)"
-	vagrant ssh -c "sudo systemctl reboot" || :
-
-vmcheck: vmbuild
-	true # start test harness here
+	@if [ -z "$(SKIP_VMBUILD)" ]; then \
+	  vagrant up && \
+	  vagrant rsync && \
+	  vagrant ssh -c "cd sync/vagrant && \
+	                  make install VERSION=$$(git describe)" && \
+	  ( vagrant ssh -c "sudo systemctl reboot" || : ) && \
+	  sleep 2; \
+	fi
 
 vmshell: vmbuild
-	sleep 3 # give time to VM to close ssh port
 	vagrant ssh
 
+# set up test environment to somewhat resemble uninstalled tests
+vmcheck: vmbuild
+	@env VMTESTS=1 \
+	    builddir="$(abs_builddir)" \
+	    topsrcdir="$(abs_top_srcdir)" \
+	    commondir="$(abs_top_srcdir)/tests/common" \
+	        sh tests/vmcheck/test.sh
+
 testenv:
 	@echo "===== ENTERING TESTENV ====="
 	test_tmpdir=$$(mktemp -d test.XXXXXX) && \

tests/common/libtest.sh

@@ -37,10 +37,9 @@ _cleanup_tmpdir () {
     fi
 }
 
-# If we're running as a local test (i.e. through `make check`), then
-# UNINSTALLEDTESTS=1. Otherwise (i.e. as an installed test), it's undefined, in
-# which case we're already in a tmpdir.
-if test -n "${UNINSTALLEDTESTS:-}" && ! test -f $PWD/.test; then
+# Create a tmpdir if we're running as a local test (i.e. through `make check`)
+# or as a `vmcheck` test, which also needs some scratch space on the host.
+if ( test -n "${UNINSTALLEDTESTS:-}" || test -n "${VMTESTS:-}" ) && ! test -f $PWD/.test; then
    test_tmpdir=$(mktemp -d test.XXXXXX)
    touch ${test_tmpdir}/.test
    trap _cleanup_tmpdir EXIT

tests/common/libvm.sh

@@ -0,0 +1,22 @@
+# Source library for installed virtualized shell script tests
+#
+# Copyright (C) 2016 Jonathan Lebon <jlebon@redhat.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+vm_cmd() {
+  $SSH "$@"
+}

tests/vmcheck/test-basic.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Copyright (C) 2016 Jonathan Lebon <jlebon@redhat.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. ${commondir}/libtest.sh
+. ${commondir}/libvm.sh
+
+set -x
+
+# try to do an upgrade -- there shouldn't be any
+vm_cmd rpm-ostree upgrade > out
+assert_file_has_content out 'No upgrade available\.'
+echo "ok no upgrade available"
+
+if vm_cmd rpm-ostree upgrade --check; then
+  assert_not_reached "upgrade --check unexpectedly passed"
+elif [ $? -ne 77 ]; then
+  assert_not_reached "upgrade --check didn't exit 77"
+fi
+echo "ok upgrade --check"

tests/vmcheck/test.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -euo pipefail
+
+# prepare ssh connection
+vagrant ssh-config > ssh-config
+echo "  ControlMaster auto" >> ssh-config
+echo "  ControlPath $PWD/ssh.sock" >> ssh-config
+echo "  ControlPersist yes" >> ssh-config
+export SSH="ssh -F $PWD/ssh-config vmcheck"
+
+# stand up ssh connection and sanity check that it all works
+if ! $SSH true &> /dev/null; then
+  echo "ERROR: A running VM is required for 'make vmcheck'."
+  exit 1
+fi
+
+LOG=${LOG:-"$PWD/vmcheck.log"}
+echo -n '' > ${LOG}
+
+testdir="$(dirname $(realpath $0))"
+cd $testdir
+
+failures=0
+for tf in $(find . -name 'test-*.sh' | sort); do
+
+    if [ -n "${TESTS+ }" ]; then
+        tfbn=$(basename "$tf" .sh)
+        tfbn=" ${tfbn#test-} "
+        if [[ " $TESTS " != *$tfbn* ]]; then
+            continue
+        fi
+    fi
+
+    bn=$(basename ${tf})
+    printf "Running $bn...\n"
+    printf "\n==== ${tf} ====\n" >> ${LOG}
+
+    # do some dirty piping to get some instant feedback and help debugging
+    if ${tf} |& tee -a ${LOG} \
+            | grep -e '^ok' --line-buffered \
+            | xargs -d '\n' -n 1 echo "  "; then
+        echo "PASS: $bn"
+    else
+        if test $? = 77; then
+            echo "SKIP: $bn"
+        else
+            echo "FAIL: $bn"
+            let "failures += 1"
+        fi
+    fi
+done
+
+# tear down ssh connection
+$SSH -O exit &>/dev/null
+
+if [ ${failures} -eq 0 ]; then
+    echo "All tests passed."
+else
+    echo "Test failures: ${failures}"
+    echo "See ${LOG} for more information."
+    exit 1
+fi