ci: check for failed services after boot

This should, hopefully, catch issues like systemd/systemd#21671
automagically.

.github/workflows/mkosi.yml

@@ -17,6 +17,11 @@ on:
 permissions:
   contents: read
 
+env:
+  # Enable debug logging in systemd, but keep udev's log level to info,
+  # since it's _very_ verbose in the QEMU task
+  KERNEL_CMDLINE: "systemd.unit=mkosi-check-and-shutdown.service !quiet systemd.log_level=debug systemd.log_target=console udev.log_level=info systemd.default_standard_output=journal+console"
+
 jobs:
   ci:
     runs-on: ubuntu-20.04
@@ -57,13 +62,20 @@ jobs:
         systemd-nspawn --version
 
     - name: Build ${{ matrix.distro }}
-      run: sudo python3 -m mkosi build
+      run: |
+        sudo python3 -m mkosi --build-environment=CI_BUILD=1 --kernel-command-line "${{ env.KERNEL_CMDLINE }}" build
 
     - name: Show ${{ matrix.distro }} image summary
       run: sudo python3 -m mkosi summary
 
     - name: Boot ${{ matrix.distro }} systemd-nspawn
-      run: sudo ./.github/workflows/test_mkosi_boot.py python3 -m mkosi boot
+      run: sudo python3 -m mkosi boot ${{ env.KERNEL_CMDLINE }}
+
+    - name: Check ${{ matrix.distro }} systemd-nspawn
+      run: sudo python3 -m mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"
 
     - name: Boot ${{ matrix.distro }} QEMU
-      run: sudo ./.github/workflows/test_mkosi_boot.py python3 -m mkosi qemu
+      run: sudo python3 -m mkosi qemu
+
+    - name: Check ${{ matrix.distro }} QEMU
+      run: sudo python3 -m mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"

.github/workflows/test_mkosi_boot.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-import pexpect
-import re
-import sys
-
-
-def run() -> None:
-    p = pexpect.spawnu(" ".join(sys.argv[1:]), logfile=sys.stdout, timeout=300)
-
-    # distro-independent root prompt
-    p.expect(re.compile("~[^#]{0,3}#"))
-    p.sendline("systemctl poweroff")
-
-    p.expect(pexpect.EOF)
-
-
-try:
-    run()
-except pexpect.EOF:
-    print("UNEXPECTED EOF")
-    sys.exit(1)
-except pexpect.TIMEOUT:
-    print("TIMED OUT")
-    sys.exit(1)

mkosi.build

@@ -110,3 +110,12 @@ if [ -n "$IMAGE_VERSION" ] ; then
     cat /tmp/os-release.tmp > "$DESTDIR"/usr/lib/os-release
     rm /tmp/os-release.tmp
 fi
+
+# If $CI_BUILD is set, copy over the CI service which executes a service check
+# after boot and then shuts down the machine
+if [ -n "$CI_BUILD" ]; then
+    mkdir -p "$DESTDIR/usr/lib/systemd/system"
+    cp -v "$SRCDIR/test/mkosi-check-and-shutdown.service" "$DESTDIR/usr/lib/systemd/system/mkosi-check-and-shutdown.service"
+    cp -v "$SRCDIR/test/mkosi-check-and-shutdown.sh" "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
+    chmod +x "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
+fi

mkosi.default.d/opensuse/10-mkosi.opensuse

@@ -23,6 +23,7 @@ BuildPackages=
         libcryptsetup-devel
         libcurl-devel
         libgcrypt-devel
+        libgnutls-devel
         libkmod-devel
         liblz4-devel
         libmicrohttpd-devel
@@ -35,8 +36,8 @@ BuildPackages=
         pciutils-devel
         pcre-devel
         python3
-        python3-lxml
         python3-Jinja2
+        python3-lxml
         qrencode-devel
         system-user-nobody
         systemd-sysvinit
@@ -61,6 +62,7 @@ Packages=
         libcrypt1
         libcryptsetup12
         libgcrypt20
+        libgnutls30
         libkmod2
         liblz4-1
         libmount1

mkosi.postinst

@@ -4,3 +4,13 @@
 if [ "$1" = "final" ] && command -v bootctl > /dev/null; then
     bootctl install
 fi
+
+# Temporary workaround until https://github.com/openSUSE/suse-module-tools/commit/158643414ddb8d8208016a5f03a4484d58944d7a
+# gets into OpenSUSE repos
+if [ "$1" = "final" ] && grep -q openSUSE /etc/os-release; then
+    if [ -e "/usr/lib/systemd/system/boot-sysctl.service" ] && \
+       ! grep -F -q 'ConditionPathExists=/boot/sysctl.conf' "/usr/lib/systemd/system/boot-sysctl.service"; then
+        mkdir -p "/etc/systemd/system/boot-sysctl.service.d/"
+        printf '[Unit]\nConditionPathExists=/boot/sysctl.conf-%%v' >"/etc/systemd/system/boot-sysctl.service.d/99-temporary-workaround.conf"
+    fi
+fi

test/mkosi-check-and-shutdown.service

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+[Unit]
+Description=Check if any service failed and then shutdown the machine
+After=multi-user.target network-online.target
+Requires=multi-user.target
+Wants=systemd-resolved.service systemd-networkd.service network-online.target
+OnFailure=poweroff.target
+OnFailureJobMode=replace-irreversibly
+
+[Service]
+Type=oneshot
+ExecStartPre=-rm -f /failed-services
+ExecStart=/usr/lib/systemd/mkosi-check-and-shutdown.sh
+ExecStartPost=systemctl poweroff --no-block

test/mkosi-check-and-shutdown.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -eux
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+systemctl --failed --no-legend | tee /failed-services
+
+# Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
+[[ ! -s /failed-services ]]
+
+: >/testok