test: Various mkosi integration test improvements

- Stop using logging module since the default output formatting is
  pretty bad. Prefer print() for now.
- Log less, logging the full mkosi command line is rather verbose,
  especially when it contains multi-line dropins.
- Streamline the journalctl command we output for debugging failed
  tests.
- Don't force usage of the disk image format.
- Don't force running without unit tests.
- Don't force disabling RuntimeBuildSources.
- Update documentation to streamline the command for running a single
  test and remove sudo as it's not required anymore.
- Improve the console output by having the test unit's output logged
  to both the journal and the console.
- Disable journal console log forwarding as we have journal forwarding
  as a better alternative.
- Delete existing journal file before running test.
- Delete journal files of succeeded tests to reduce disk usage.
- Rename system_mkosi target to just mkosi
- Pass in mkosi source directory explicitly to accomodate arbitrary
  build directory locations.
- Add test interactive debugging if stdout is connected to a tty
- Stop explicitly using the 'system' image since it'll likely be
  dropped soon.
- Only forward journal if we're not running in debugging mode.
- Stop using testsuite.target and instead just add the necessary
  extras to the main testsuite unit via the credential dropin.
- Override type to idle so test output is not interleaved with
  status output.
- Don't build mkosi target by default
- Always add the mkosi target if mkosi is found
- Remove dependency of the integration tests on the mkosi target
  as otherwise the image is always built, even though we configure
  it to not be built by default.
- Move mkosi output, cache and build directory into build/ so that
  invocations from meson and regular invocations share the same
  directories.
- Various aesthetic cleanups.

meson.build

@@ -2573,13 +2573,27 @@ endif
 
 #####################################################################
 
-if get_option('integration-tests') != false
+mkosi = find_program('mkosi', required : false)
-        system_mkosi = custom_target('system_mkosi',
+if mkosi.found()
+        custom_target('mkosi',
                 build_always_stale : true,
-                output : 'system',
+                build_by_default: false,
                 console : true,
-                command : ['mkosi', '-C', meson.project_source_root(), '--image=system', '--format=disk', '--output-dir', meson.project_build_root() / '@OUTPUT@', '--without-tests', '-fi', 'build'],
+                output : '.',
-                depends : [executables_by_name['bootctl'], executables_by_name['systemd-measure'], executables_by_name['systemd-repart'], ukify],
+                command : [
+                        'mkosi',
+                        '--directory', meson.current_source_dir(),
+                        '--output-dir', meson.current_build_dir() / 'mkosi.output',
+                        '--cache-dir', meson.current_build_dir() / 'mkosi.cache',
+                        '--build-dir', meson.current_build_dir() / 'mkosi.builddir',
+                        '--force',
+                        'build'
+                ],
+                depends : public_programs + [
+                        executables_by_name['systemd-journal-remote'],
+                        executables_by_name['systemd-measure'],
+                        ukify,
+                ],
         )
 endif
 

mkosi.conf

@@ -5,9 +5,9 @@
 MinimumVersion=23~devel
 
 [Output]
-@OutputDirectory=mkosi.output
+@OutputDirectory=build/mkosi.output
-@BuildDirectory=mkosi.builddir
+@BuildDirectory=build/mkosi.builddir
-@CacheDirectory=mkosi.cache
+@CacheDirectory=build/mkosi.cache
 
 [Content]
 # Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_.
@@ -20,8 +20,6 @@ BuildSourcesEphemeral=yes
 KernelCommandLine=systemd.crash_shell
                   systemd.log_level=debug,console:info
                   systemd.log_ratelimit_kmsg=0
-                  systemd.journald.forward_to_console
-                  systemd.journald.max_level_console=warning
                   # Disable the kernel's ratelimiting on userspace logging to kmsg.
                   printk.devkmsg=on
                   # Make sure /sysroot is mounted rw in the initrd.

test/README.testsuite

@@ -33,14 +33,24 @@ enable integration tests and options for required commands with the following:
 
 $ meson configure build -Dintegration-tests=true -Dremote=enabled -Dopenssl=enabled -Dblkid=enabled -Dtpm2=enabled
 
-Once enabled the integration tests can be run with:
+Once enabled, first build the integration test image:
 
-$ sudo meson test -C build/ --suite integration-tests --num-processes "$((nproc / 2))"
+$ meson compile -C build mkosi
+
+After the image has been built, the integration tests can be run with:
+
+$ meson test -C build/ --suite integration-tests --num-processes "$(($(nproc) / 2))"
 
 As usual, specific tests can be run in meson by appending the name of the test
 which is usually the name of the directory e.g.
 
-$ sudo meson test -C build/ --suite integration-tests --num-processes "$((nproc / 2))" TEST-01-BASIC
+$ meson test -C build/ -v TEST-01-BASIC
+
+Due to limitations in meson, the integration tests do not yet depend on the mkosi target, which means the
+mkosi target has to be manually rebuilt before running the integration tests. To rebuild the image and rerun
+a test, the following command can be used:
+
+$ meson compile -C build mkosi && meson test -C build -v TEST-01-BASIC
 
 See `meson introspect build --tests` for a list of tests.
 

test/integration-test-wrapper.py

@@ -0,0 +1,130 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+'''Test wrapper command for driving integration tests.
+
+Note: This is deliberately rough and only intended to drive existing tests
+with the expectation that as part of formally defining the API it will be tidy.
+
+'''
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+
+EMERGENCY_EXIT_DROPIN = """\
+[Unit]
+Wants=emergency-exit.service
+"""
+
+
+EMERGENCY_EXIT_SERVICE = """\
+[Unit]
+DefaultDependencies=no
+Conflicts=shutdown.target
+Conflicts=rescue.service
+Before=shutdown.target
+Before=rescue.service
+FailureAction=exit
+
+[Service]
+ExecStart=false
+"""
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--meson-source-dir', required=True, type=Path)
+    parser.add_argument('--meson-build-dir', required=True, type=Path)
+    parser.add_argument('--test-name', required=True)
+    parser.add_argument('--test-number', required=True)
+    parser.add_argument('mkosi_args', nargs="*")
+    args = parser.parse_args()
+
+    test_unit = f"testsuite-{args.test_number}.service"
+
+    dropin = textwrap.dedent(
+        """\
+        [Unit]
+        After=multi-user.target network.target
+        Requires=multi-user.target
+
+        [Service]
+        StandardOutput=journal+console
+        """
+    )
+
+    if not sys.stderr.isatty():
+        dropin += textwrap.dedent(
+            """
+            [Unit]
+            SuccessAction=exit
+            FailureAction=exit
+            """
+        )
+
+        journal_file = (args.meson_build_dir / (f"test/journal/{args.test_name}.journal")).absolute()
+        journal_file.unlink(missing_ok=True)
+    else:
+        journal_file = None
+
+    cmd = [
+        'mkosi',
+        '--directory', os.fspath(args.meson_source_dir),
+        '--output-dir', os.fspath(args.meson_build_dir / 'mkosi.output'),
+        '--extra-search-path', os.fspath(args.meson_build_dir),
+        '--machine', args.test_name,
+        '--ephemeral',
+        *(['--forward-journal', journal_file] if journal_file else []),
+        *(
+            [
+                '--credential',
+                f"systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}",
+                '--credential',
+                f"systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}",
+                '--kernel-command-line-extra=systemd.mask=serial-getty@.service',
+            ]
+            if not sys.stderr.isatty()
+            else []
+        ),
+        '--credential',
+        f"systemd.unit-dropin.{test_unit}={shlex.quote(dropin)}",
+        '--append',
+        '--kernel-command-line-extra',
+        ' '.join([
+            'systemd.hostname=H',
+            f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-{args.test_number}.units:/usr/lib/systemd/tests/testdata/units:",
+            f"systemd.unit={test_unit}",
+        ]),
+        *args.mkosi_args,
+        'qemu',
+    ]
+
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        if e.returncode != 77 and journal_file:
+            cmd = [
+                'journalctl',
+                '--no-hostname',
+                '-o', 'short-monotonic',
+                '--file', journal_file,
+                '-u', test_unit,
+                '-p', 'info',
+            ]
+            print("Test failed, relevant logs can be viewed with: \n\n"
+                  f"{shlex.join(str(a) for a in cmd)}\n", file=sys.stderr)
+        exit(e.returncode)
+
+    # Do not keep journal files for tests that don't fail.
+    if journal_file:
+        journal_file.unlink(missing_ok=True)
+
+
+if __name__ == '__main__':
+    main()

test/integration_test_wrapper.py

@@ -1,134 +0,0 @@
-#!/usr/bin/python3
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-'''Test wrapper command for driving integration tests.
-
-Note: This is deliberately rough and only intended to drive existing tests
-with the expectation that as part of formally defining the API it will be tidy.
-
-'''
-
-import argparse
-import logging
-import os
-from pathlib import Path
-import shlex
-import subprocess
-
-
-TEST_EXIT_DROPIN = """\
-[Unit]
-SuccessAction=exit
-FailureAction=exit
-"""
-
-
-EMERGENCY_EXIT_DROPIN = """\
-[Unit]
-Wants=emergency-exit.service
-"""
-
-
-EMERGENCY_EXIT_SERVICE = """\
-[Unit]
-DefaultDependencies=no
-Conflicts=shutdown.target
-Conflicts=rescue.service
-Before=shutdown.target
-Before=rescue.service
-FailureAction=exit
-
-[Service]
-ExecStart=false
-"""
-
-
-parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument('--test-name', required=True)
-parser.add_argument('--mkosi-image-name', required=True)
-parser.add_argument('--mkosi-output-path', required=True, type=Path)
-parser.add_argument('--test-number', required=True)
-parser.add_argument('--no-emergency-exit',
-                    dest='emergency_exit', default=True, action='store_false',
-                    help="Disable emergency exit drop-ins for interactive debugging")
-parser.add_argument('mkosi_args', nargs="*")
-
-def main():
-    logging.basicConfig(level=logging.DEBUG)
-    args = parser.parse_args()
-
-    test_unit_name = f"testsuite-{args.test_number}.service"
-    # Machine names shouldn't have / since it's used as a file name
-    # and it must be a valid hostname so 64 chars max
-    machine_name = args.test_name.replace('/', '_')[:64]
-
-    logging.debug(f"test name: {args.test_name}\n"
-                  f"test number: {args.test_number}\n"
-                  f"image: {args.mkosi_image_name}\n"
-                  f"mkosi output path: {args.mkosi_output_path}\n"
-                  f"mkosi args: {args.mkosi_args}\n"
-                  f"emergency exit: {args.emergency_exit}")
-
-    journal_file = Path(f"{machine_name}.journal").absolute()
-    logging.info(f"Capturing journal to {journal_file}")
-
-    mkosi_args = [
-        'mkosi',
-        '--directory', Path('..').resolve(),
-        '--output-dir', args.mkosi_output_path.absolute(),
-        '--machine', machine_name,
-        '--image', args.mkosi_image_name,
-        '--format=disk',
-        '--runtime-build-sources=no',
-        '--ephemeral',
-        '--forward-journal', journal_file,
-        *(
-            [
-                '--credential',
-                f"systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)} "
-                f"systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}",
-            ]
-            if args.emergency_exit
-            else []
-        ),
-        f"--credential=systemd.unit-dropin.{test_unit_name}={shlex.quote(TEST_EXIT_DROPIN)}",
-        '--append',
-        '--kernel-command-line-extra',
-        ' '.join([
-            'systemd.hostname=H',
-            f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-{args.test_number}.units:/usr/lib/systemd/tests/testdata/units:",
-            'systemd.unit=testsuite.target',
-            f"systemd.wants={test_unit_name}",
-        ]),
-        *args.mkosi_args,
-    ]
-
-    mkosi_args += ['qemu']
-
-    logging.debug(f"Running {shlex.join(os.fspath(a) for a in mkosi_args)}")
-
-    try:
-        subprocess.run(mkosi_args, check=True)
-    except subprocess.CalledProcessError as e:
-        if e.returncode not in (0, 77):
-            suggested_command = [
-                'journalctl',
-                '--all',
-                '--no-hostname',
-                '-o', 'short-monotonic',
-                '--file', journal_file,
-                f"_SYSTEMD_UNIT={test_unit_name}",
-                '+', f"SYSLOG_IDENTIFIER=testsuite-{args.test_number}.sh",
-                '+', 'PRIORITY=4',
-                '+', 'PRIORITY=3',
-                '+', 'PRIORITY=2',
-                '+', 'PRIORITY=1',
-                '+', 'PRIORITY=0',
-            ]
-            logging.info("Test failed, relevant logs can be viewed with: "
-                         f"{shlex.join(os.fspath(a) for a in suggested_command)}")
-        exit(e.returncode)
-
-
-if __name__ == '__main__':
-    main()

test/meson.build

@@ -334,21 +334,19 @@ endif
 
 ############################################################
 
-if get_option('integration-tests') != false
+if get_option('integration-tests')
-        integration_test_wrapper = find_program('integration_test_wrapper.py')
+        if not mkosi.found()
+                error('Could not find mkosi which is required to run the integration tests')
+        endif
+
+        integration_test_wrapper = find_program('integration-test-wrapper.py')
         integration_tests = {
                 '01': 'TEST-01-BASIC',
                 '02': 'TEST-02-UNITTESTS',
         }
         foreach test_number, dirname : integration_tests
-                test_unit_name = f'testsuite-@test_number@.service'
                 test_params = {
-                        'test_name' : dirname,
-                        'mkosi_image_name' : 'system',
-                        'mkosi_output_path' : system_mkosi,
-                        'test_number' : test_number,
                         'mkosi_args' : [],
-                        'depends' : [system_mkosi],
                         'timeout' : 600,
                 }
 
@@ -358,16 +356,22 @@ if get_option('integration-tests') != false
                 if fs.exists(dirname / 'meson.build')
                         subdir(dirname)
                 endif
-                args = ['--test-name', test_params['test_name'],
+
-                        '--mkosi-image-name', test_params['mkosi_image_name'],
+                args = [
-                        '--mkosi-output-path', test_params['mkosi_output_path'],
+                        '--meson-source-dir', meson.project_source_root(),
-                        '--test-number', test_params['test_number']]
+                        '--meson-build-dir', meson.project_build_root(),
-                args += ['--'] + test_params['mkosi_args']
+                        '--test-name', dirname,
-                test(test_params['test_name'],
+                        '--test-number', test_number,
+                        '--',
+                ] + test_params['mkosi_args']
+
+                # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt
+                # on every "ninja -C build". Instead, the mkosi target has to be rebuilt manually before
+                # running the integration tests with mkosi.
+                test(dirname,
                      integration_test_wrapper,
                      env: test_env,
                      args : args,
-                     depends : test_params['depends'],
                      timeout : test_params['timeout'],
                      suite : 'integration-tests')
         endforeach