test: Add a way to quickly iterate on an integration test

Rebuilding the integration test every time is very slow. Let's
introduce a way to iterate on an integration test without rebuilding
the image every time. By making a btrfs snapshot before we run the
integration test, we can then systemctl soft-reboot after running
the test to restore the rootfs to a pristine state before running
the test again.

As /run/nextroot will get nuked on reboot or soft-reboot, we introduce
a tmpfiles snippet to make sure it is recreated every (soft-)reboot
and adapt the existing tests to deal with this new symlink.

mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+L /run/nextroot - - - - /snapshot

test/README.testsuite

@@ -86,6 +86,45 @@ mkosi in the systemd reposistory, so any local modifications to the mkosi
 configuration (e.g. in `mkosi.local.conf`) are automatically picked up and used
 by the integration tests as well.
 
+## Iterating on an integration test
+
+To iterate on an integration test, let's first get a shell in the integration test environment by running
+the following:
+
+```shell
+$ meson compile -C build mkosi && SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 meson test -C build --no-rebuild -i TEST-01-BASIC
+```
+
+This will get us a shell in the integration test environment after booting the machine without running the
+integration test itself. After booting, we can verify the integration test passes by running it manually,
+for example with `systemctl start TEST-01-BASIC`.
+
+Now you can extend the test in whatever way you like to add more coverage of existing features or to add
+coverage for a new feature. Once you've finished writing the logic and want to rerun the test, run the
+the following on the host:
+
+```shell
+$ mkosi -t none
+```
+
+This will rebuild the distribution packages without rebuilding the entire integration test image. Next, run
+the following in the integration test machine:
+
+```shell
+$ systemctl soft-reboot
+$ systemctl start TEST-01-BASIC
+```
+
+A soft-reboot is required to make sure all the leftover state from the previous run of the test is cleaned
+up by soft-rebooting into the btrfs snapshot we made before running the test. After the soft-reboot,
+re-running the test will first install the new packages we just built, make a new snapshot and finally run
+the test again. You can keep running the loop of `mkosi -t none`, `systemctl soft-reboot` and
+`systemctl start ...` until the changes to the integration test are working.
+
+If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`),
+there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to
+rerun the test.
+
 ## Running the integration tests the old fashioned way
 
 The extended testsuite only works with UID=0. It consists of the subdirectories

test/integration-test-setup.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+case "$1" in
+    setup)
+        if [[ -f "$STATE_DIRECTORY/inprogress" ]]; then
+            exit 0
+        fi
+
+        if [[ -d /snapshot ]]; then
+            echo "Run systemctl soft-reboot first to make sure the test runs within a pristine rootfs" >&2
+            exit 1
+        fi
+
+        . /usr/lib/os-release
+
+        if test -n "$(shopt -s nullglob; echo /work/build/*.{rpm,deb,pkg.tar})"; then
+            case "$ID" in
+                arch)
+                    pacman --upgrade --needed --noconfirm /work/build/*.pkg.tar
+                    ;;
+                debian|ubuntu)
+                    apt-get install /work/build/*.deb
+                    ;;
+                opensuse*)
+                    zypper --non-interactive install --allow-unsigned-rpm /work/build/*.rpm
+                    ;;
+                centos|fedora)
+                    dnf upgrade --assumeyes --disablerepo="*" /work/build/*.rpm
+                    ;;
+                *)
+                    echo "Unknown distribution $ID" >&2
+                    exit 1
+            esac
+        fi
+
+        # TODO: Use a proper flat btrfs subvolume layout once we can create subvolumes without privileged in
+        # systemd-repart (see https://github.com/systemd/systemd/pull/33498). Until that's possible, we nest
+        # snapshots within each other.
+        if command -v btrfs >/dev/null && [[ "$(stat --file-system --format %T /)" == "btrfs" ]]; then
+            btrfs subvolume snapshot / /snapshot
+        fi
+
+        touch "$STATE_DIRECTORY/inprogress"
+        ;;
+    finalize)
+        # If we're rebooting, the test does a reboot as part of its execution and we shouldn't remove /inprogress.
+        if ! [[ "$(systemctl list-jobs)" =~ reboot.target|kexec.target|soft-reboot.target ]]; then
+            rm -f "$STATE_DIRECTORY/inprogress"
+        fi
+        ;;
+    *)
+        echo "Unknown verb $1" >&2
+        exit 1
+esac

test/meson.build

@@ -142,9 +142,11 @@ endif
 ############################################################
 
 if install_tests
-        install_data('run-unit-tests.py',
+        foreach script : ['integration-test-setup.sh', 'run-unit-tests.py']
+                install_data(script,
                              install_mode : 'rwxr-xr-x',
                              install_dir : testsdir)
+        endforeach
 endif
 
 ############################################################

test/test.service.in

@@ -4,9 +4,12 @@ Description=%N
 Wants=basic.target network.target @wants@
 After=basic.target network.target @after@
 Before=getty-pre.target
+StateDirectory=%N
 
 [Service]
 ExecStartPre=rm -f /failed /testok
+ExecStartPre=/usr/lib/systemd/tests/integration-test-setup.sh setup
 ExecStart=@command@
+ExecStopPost=/usr/lib/systemd/tests/integration-test-setup.sh finalize
 Type=oneshot
 MemoryAccounting=@memory-accounting@

test/units/TEST-09-REBOOT.sh

@@ -17,7 +17,11 @@ systemd-cat journalctl --list-boots
 run_subtests
 
 if [[ "$REBOOT_COUNT" -lt "$NUM_REBOOT" ]]; then
+    SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT=1
+    export SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT
     systemctl_final reboot
+    # Now block until the reboot killing spree kills us.
+    exec sleep infinity
 elif [[ "$REBOOT_COUNT" -gt "$NUM_REBOOT" ]]; then
     assert_not_reached
 fi

test/units/TEST-82-SOFTREBOOT.sh

@@ -19,6 +19,21 @@ at_exit() {
 
 trap at_exit EXIT
 
+# Because this test tests soft-reboot, we have to get rid of the symlink we put at
+# /run/nextroot to allow rebooting into the previous snapshot if the test fails for
+# the duration of the test. However, let's make sure we put the symlink back in place
+# if the test fails.
+if [[ -L /run/nextroot ]]; then
+    at_error() {
+        mountpoint -q /run/nextroot && umount -R /run/nextroot
+        rm -rf /run/nextroot
+        ln -sf /snapshot /run/nextroot
+    }
+
+    trap at_error ERR
+    rm -f /run/nextroot
+fi
+
 systemd-analyze log-level debug
 
 export SYSTEMD_LOG_LEVEL=debug