test: rework how images are created

Before, we'd create a separate image for each test, in
/var/tmp/systemd-test.XXXXX/rootdisk.img. Most of the images
where very similar, except that each one had some unit files installed
specifically for the test. The installation of those custom unit files
was removed in previous commits (all the unit files are always installed).

The new approach is to only create as few distinct images as possible.
We have:
default.img: the "normal" image suitable for almost all the tests
basic.img: the same as default image but doesn't mask any services
cryptsetup.img: p2 is used for encrypted /var
badid.img: /etc/machine-id is overwritten with stuff
selinux.img: with selinux added for fun and fun
and a few others:

ls -l build/test/*img
lrwxrwxrwx 1 root root 38 Mar 21 21:23 build/test/badid.img -> /var/tmp/systemd-test.PJFFeo/badid.img
lrwxrwxrwx 1 root root 38 Mar 21 21:17 build/test/basic.img -> /var/tmp/systemd-test.na0xOI/basic.img
lrwxrwxrwx 1 root root 43 Mar 21 21:18 build/test/cryptsetup.img -> /var/tmp/systemd-test.Tzjv06/cryptsetup.img
lrwxrwxrwx 1 root root 40 Mar 21 21:19 build/test/default.img -> /var/tmp/systemd-test.EscAsS/default.img
lrwxrwxrwx 1 root root 39 Mar 21 21:22 build/test/nspawn.img -> /var/tmp/systemd-test.HSebKo/nspawn.img
lrwxrwxrwx 1 root root 40 Mar 21 21:20 build/test/selinux.img -> /var/tmp/systemd-test.daBjbx/selinux.img
lrwxrwxrwx 1 root root 39 Mar 21 21:21 build/test/test08.img -> /var/tmp/systemd-test.OgnN8Z/test08.img

I considered trying to use the same image everywhere. It would probably be
possible, but it would be very brittle. By using separate images where it is
necessary we keep various orthogonal modifications independent.

The way that images are cached is complicated by the fact that we still
want to keep them in /var/tmp. Thus, an image is created on first use and
linked to from build/test/ so it can be found by other tests.

Tests cannot be run in parallel. I think that is an acceptable limitation.
Creation of the images was probably taking more resources then the actual
tests, so we should be better off anyway.

test/TEST-01-BASIC/test.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="Basic systemd setup"
+IMAGE_NAME="basic"
 RUN_IN_UNPRIVILEGED_CONTAINER=${RUN_IN_UNPRIVILEGED_CONTAINER:-yes}
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay

test/TEST-02-CRYPTSETUP/test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="cryptsetup systemd setup"
+IMAGE_NAME="cryptsetup"
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
@@ -23,8 +24,7 @@ check_result_qemu() {
     return $ret
 }
 
-
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
     echo -n test >$TESTDIR/keyfile
     cryptsetup -q luksFormat --pbkdf pbkdf2 --pbkdf-force-iterations 1000 ${LOOPDEV}p2 $TESTDIR/keyfile

test/TEST-06-SELINUX/test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="SELinux tests"
+IMAGE_NAME="selinux"
 TEST_NO_NSPAWN=1
 
 # Requirements:
@@ -15,7 +16,7 @@ test -f /usr/share/selinux/devel/include/system/systemd.if || exit 0
 SETUP_SELINUX=yes
 KERNEL_APPEND="$KERNEL_APPEND selinux=1 security=selinux"
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay

test/TEST-08-ISSUE-2730/test.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="https://github.com/systemd/systemd/issues/2730"
+IMAGE_NAME="test08"
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 QEMU_TIMEOUT=300
 FSTYPE=ext4
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay

test/TEST-13-NSPAWN-SMOKE/test.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="systemd-nspawn smoke test"
+IMAGE_NAME=nspawn
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay

test/TEST-14-MACHINE-ID/test.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="/etc/machine-id testing"
+IMAGE_NAME=badid
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay

test/TEST-21-SYSUSERS/test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="Sysuser-related tests"
-
+IMAGE_NAME=sysusers
 . $TEST_BASE_DIR/test-functions
 
 test_setup() {

test/TEST-24-UNIT-TESTS/test.sh

@@ -24,13 +24,13 @@ check_result_nspawn() {
     fi
     cp -a $TESTDIR/$1/var/log/journal $TESTDIR
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
+    umount_initdir
     return $_ret
 }
 
 check_result_qemu() {
     local _ret=1
-    mkdir -p $initdir
-    mount ${LOOPDEV}p1 $initdir
+    mount_initdir
     [[ -e $initdir/testok ]] && _ret=0
     if [[ -s $initdir/failed ]]; then
         _ret=$(($_ret+1))
@@ -47,7 +47,7 @@ check_result_qemu() {
         fi
     fi
     cp -a $initdir/var/log/journal $TESTDIR
-    umount $initdir
+    umount_initdir
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
     return $_ret
 }

test/run-integration-tests.sh

@@ -4,8 +4,10 @@ set -e
 BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)"
 if [ $# -gt 0 ]; then
     args="$@"
+    do_clean=0
 else
-    args="clean setup run clean-again"
+    args="setup run"
+    do_clean=1
 fi
 
 ninja -C "$BUILD_DIR"
@@ -16,6 +18,13 @@ COUNT=0
 FAILURES=0
 
 cd "$(dirname "$0")"
+
+if [ $do_clean = 1 ]; then
+    for TEST in TEST-??-* ; do
+        ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean )
+    done
+fi
+
 for TEST in TEST-??-* ; do
     COUNT=$(($COUNT+1))
 
@@ -31,6 +40,12 @@ for TEST in TEST-??-* ; do
     [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1))
 done
 
+if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then
+    for TEST in TEST-??-* ; do
+        ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean-again )
+    done
+fi
+
 echo ""
 
 for TEST in ${!results[@]}; do

test/test-functions

@@ -16,6 +16,7 @@ TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
 QEMU_MEM="${QEMU_MEM:-512M}"
+IMAGE_NAME=${IMAGE_NAME:-default}
 
 # Decide if we can (and want to) run QEMU with KVM acceleration.
 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
@@ -136,6 +137,7 @@ DEBUGTOOLS=(
 
 STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
 STATEFILE="$STATEDIR/.testdir"
+IMAGESTATEDIR="$STATEDIR/.."
 TESTLOG="$STATEDIR/test.log"
 
 is_built_with_asan() {
@@ -276,6 +278,9 @@ run_qemu() {
 
     find_qemu_bin || return 1
 
+    # Umount initdir to avoid concurrent access to the filesystem
+    umount_initdir
+
     local _cgroup_args
     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
         _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
@@ -320,7 +325,7 @@ $KERNEL_APPEND \
 -m $QEMU_MEM \
 -nographic \
 -kernel $KERNEL_BIN \
--drive format=raw,cache=unsafe,file=${TESTDIR}/rootdisk.img \
+-drive format=raw,cache=unsafe,file=${IMAGESTATEDIR}/${IMAGE_NAME}.img \
 $QEMU_OPTIONS \
 "
 
@@ -631,7 +636,7 @@ install_systemd() {
     # and it could fill the available space
     strip_binaries
 
-   [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
+    [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
 
     # enable debug logging in PID1
     echo LogLevel=debug >> $initdir/etc/systemd/system.conf
@@ -658,19 +663,26 @@ install_missing_libraries() {
 }
 
 create_empty_image() {
+    if [ -z "$IMAGE_NAME" ]; then
+        echo "create_empty_image: \$IMAGE_NAME not set"
+        exit 1
+    fi
+
     local _size=500
     if [[ "$STRIP_BINARIES" = "no" ]]; then
         _size=$((4*_size))
     fi
 
-    echo "Setting up $TESTDIR/rootdisk.img (${_size} MB)"
+    image="${TESTDIR}/${IMAGE_NAME}.img"
+    public="$IMAGESTATEDIR/${IMAGE_NAME}.img"
+    echo "Setting up $public (${_size} MB)"
 
-    rm -f "$TESTDIR/rootdisk.img"
+    rm -f "$image" "$public"
     # Create the blank file to use as a root filesystem
-    truncate -s "${_size}M" "$TESTDIR/rootdisk.img"
-    LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
+    truncate -s "${_size}M" "$image"
+    LOOPDEV=$(losetup --show -P -f "$image")
     [ -b "$LOOPDEV" ] || return 1
-    echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
+    echo "LOOPDEV=$LOOPDEV" >>$STATEFILE
     sfdisk "$LOOPDEV" <<EOF
 ,$((_size-50))M
 ,
@@ -685,13 +697,37 @@ EOF
         dfatal "Failed to mkfs -t ${FSTYPE}"
         exit 1
     fi
+
+    # the image is created, let's expose it
+    ln -vs "$(realpath $image)" "$public"
+}
+
+mount_initdir() {
+    if [ -z "${LOOPDEV}" ]; then
+        image="${TESTDIR}/${IMAGE_NAME}.img"
+        LOOPDEV=$(losetup --show -P -f "$image")
+        [ -b "$LOOPDEV" ] || return 1
+        echo "LOOPDEV=$LOOPDEV" >>$STATEFILE
+    fi
+
+    mkdir -p $initdir
+    mount ${LOOPDEV}p1 $initdir
+    TEST_SETUP_CLEANUP_ROOTDIR=1
+}
+
+umount_initdir() {
+    _umount_dir $initdir
+    if [[ $LOOPDEV && -b $LOOPDEV ]]; then
+        ddebug "losetup -d $LOOPDEV"
+        losetup -d $LOOPDEV
+    fi
+    LOOPDEV=
+    sed -i /LOOPDEV=/d $STATEFILE
 }
 
 create_empty_image_rootdir() {
     create_empty_image
-    mkdir -p $initdir
-    mount ${LOOPDEV}p1 $initdir
-    TEST_SETUP_CLEANUP_ROOTDIR=1
+    mount_initdir
 }
 
 check_asan_reports() {
@@ -744,14 +780,14 @@ check_result_nspawn() {
     test -s $TESTDIR/failed && ret=$(($ret+1))
     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
     check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
+    umount_initdir
     return $ret
 }
 
 # can be overridden in specific test
 check_result_qemu() {
     local ret=1
-    mkdir -p $initdir
-    mount ${LOOPDEV}p1 $initdir
+    mount_initdir
     [[ -e $initdir/testok ]] && ret=0
     [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
     cp -a $initdir/var/log/journal $TESTDIR
@@ -1042,6 +1078,10 @@ has_user_dbus_socket() {
 }
 
 setup_nspawn_root() {
+    if [ -z "${initdir}" ]; then
+        dfatal "\$initdir not defined"
+        exit 1
+    fi
     rm -fr $TESTDIR/nspawn-root
     ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
     cp -ar $initdir $TESTDIR/nspawn-root
@@ -1110,7 +1150,10 @@ import_testdir() {
             mkdir -p "$TESTDIR"
         fi
 
-        echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
+        cat >$STATEFILE<<EOF
+TESTDIR="$TESTDIR"
+LOOPDEV="$LOOPDEV"
+EOF
         export TESTDIR
     fi
 }
@@ -1866,13 +1909,10 @@ _test_cleanup() {
     # (post-test) cleanup should always ignore failure and cleanup as much as possible
     (
         set +e
-        _umount_dir $initdir
-        if [[ $LOOPDEV && -b $LOOPDEV ]]; then
-            ddebug "losetup -d $LOOPDEV"
-            losetup -d $LOOPDEV
-        fi
-        rm -fr "$TESTDIR"
-        rm -f "$STATEFILE"
+        umount_initdir
+        rm -vf "${IMAGESTATEDIR}/${IMAGE_NAME}.img"
+        rm -vfr "$TESTDIR"
+        rm -vf "$STATEFILE"
     ) || :
 }
 
@@ -1881,12 +1921,7 @@ test_cleanup() {
     _test_cleanup
 }
 
-test_setup() {
-    if type -P meson >/dev/null && [[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
-        dfatal "Needs to be built with -Dinstall-tests=true"
-        exit 1
-    fi
-
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
@@ -1895,6 +1930,27 @@ test_setup() {
         setup_basic_environment
         mask_supporting_services
     )
+}
+
+test_setup() {
+    if type -P meson >/dev/null && [[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
+        dfatal "Needs to be built with -Dinstall-tests=true"
+        exit 1
+    fi
+
+    image="${TESTDIR}/${IMAGE_NAME}.img"
+    public="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
+    if [ -e "$image" ]; then
+        echo "Reusing existing image $PWD/$image → $(realpath $image)"
+        mount_initdir
+    elif [ -e "$public" ]; then
+        echo "Reusing existing cached image $PWD/$public → $(realpath $public)"
+        ln -s "$(realpath $public)" "$image"
+        mount_initdir
+    else
+        test_create_image
+    fi
+
     setup_nspawn_root
 }