ci: Implement coverage on top of mkosi

.github/workflows/coverage.yml

@@ -0,0 +1,145 @@
+---
+# SPDX-License-Identifier: LGPL-2.1-or-later
+name: coverage
+
+on:
+  schedule:
+    # Calculate coverage daily at midnight
+    - cron:  '0 0 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  coverage:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: systemd/mkosi@07ef37c4c0dad5dfc6cec86c967a7600df1cd88c
+
+      # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
+      # immediately, we remove the files in the background. However, we first move them to a different location
+      # so that nothing tries to use anything in these directories anymore while we're busy deleting them.
+      - name: Free disk space
+        run: |
+          sudo mv /usr/local /usr/local.trash
+          sudo mv /opt/hostedtoolcache /opt/hostedtoolcache.trash
+          sudo systemd-run rm -rf /usr/local.trash /opt/hostedtoolcache.trash
+
+      - name: Btrfs
+        run: |
+          truncate --size=100G btrfs.raw
+          mkfs.btrfs btrfs.raw
+          sudo mkdir /mnt/mkosi
+          LOOP="$(sudo losetup --find --show --direct-io=on btrfs.raw)"
+          sudo mount "$LOOP" /mnt/mkosi --options compress=zstd:1,user_subvol_rm_allowed,noatime,discard=async,space_cache=v2
+          sudo chown "$(id -u):$(id -g)" /mnt/mkosi
+          mkdir /mnt/mkosi/tmp
+          echo "TMPDIR=/mnt/mkosi/tmp" >>"$GITHUB_ENV"
+          ln -s /mnt/mkosi/build build
+
+      - name: Configure
+        run: |
+          # XXX: drop after the HyperV bug that breaks secure boot KVM guests is solved
+          sed -i "s/'firmware'\s*:\s*'auto'/'firmware' : 'uefi'/g" test/*/meson.build
+
+          tee mkosi.local.conf <<EOF
+          [Distribution]
+          Distribution=arch
+
+          [Build]
+          ToolsTree=default
+          ToolsTreeDistribution=arch
+          UseSubvolumes=yes
+          WithTests=no
+
+          WorkspaceDirectory=$TMPDIR
+          PackageCacheDirectory=$TMPDIR/cache
+
+          Environment=
+                  # Build debuginfo packages since we'll be publishing the packages as artifacts.
+                  WITH_DEBUG=1
+                  CFLAGS=-Og
+                  MESON_OPTIONS=--werror
+                  COVERAGE=1
+
+          [Host]
+          QemuMem=4G
+          EOF
+
+      - name: Generate secure boot key
+        run: mkosi --debug genkey
+
+      - name: Show image summary
+        run: mkosi summary
+
+      - name: Build tools tree
+        run: mkosi -f sandbox true
+
+      - name: PATH
+        run: echo "$PATH"
+
+      - name: Configure meson
+        run: mkosi sandbox meson setup --buildtype=debugoptimized -Dintegration-tests=true build
+
+      - name: Build image
+        run: sudo --preserve-env mkosi sandbox meson compile -C build mkosi
+
+      - name: Initial coverage report
+        run: |
+          mkdir -p build/test/coverage
+          mkosi sandbox \
+            lcov \
+            --directory build/mkosi.builddir/arch~rolling~x86-64 \
+            --capture \
+            --initial \
+            --exclude "*.gperf" \
+            --output-file build/test/coverage/initial.coverage-info \
+            --base-directory src/ \
+            --ignore-errors source \
+            --no-external \
+            --substitute "s#src/src#src#g"
+
+      - name: Run integration tests
+        run: |
+          sudo --preserve-env \
+            mkosi sandbox \
+            meson test \
+            -C build \
+            --no-rebuild \
+            --suite integration-tests \
+            --print-errorlogs \
+            --no-stdsplit \
+            --num-processes "$(($(nproc) - 1))" \
+            --timeout-multiplier 2 \
+            --max-lines 300
+
+      - name: Archive failed test journals
+        uses: actions/upload-artifact@v4
+        if: failure() && (github.repository == 'systemd/systemd' || github.repository == 'systemd/systemd-stable')
+        with:
+          name: ci-coverage-${{ github.run_id }}-${{ github.run_attempt }}-arch-rolling-failed-test-journals
+          path: |
+            build/test/journal/*.journal
+            build/meson-logs/*
+          retention-days: 7
+
+      - name: Combine coverage reports
+        run: |
+          lcov_args=()
+
+          while read -r file; do
+              lcov_args+=(--add-tracefile "${file}")
+          done < <(find build/test/coverage -name "TEST-*.coverage-info")
+
+          mkosi sandbox lcov --ignore-errors inconsistent,inconsistent "${lcov_args[@]}" --output-file build/test/coverage/everything.coverage-info
+
+      - name: List coverage report
+        run: mkosi sandbox lcov --ignore-errors inconsistent,inconsistent --list build/test/coverage/everything.coverage-info
+
+      - name: Coveralls
+        uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8
+        if: github.repository == 'systemd/systemd' || github.repository == 'systemd/systemd-stable'
+        with:
+          file: build/test/coverage/everything.coverage-info

mkosi.conf

@@ -22,6 +22,7 @@ PassEnvironment=
         SYSEXT
         WITH_DEBUG
         ASAN_OPTIONS
+        COVERAGE
 
 [Output]
 RepartDirectories=mkosi.repart
@@ -150,3 +151,4 @@ QemuKvm=yes
 
 [Include]
 Include=%D/mkosi.sanitizers
+        %D/mkosi.coverage

mkosi.conf.d/05-tools/mkosi.conf

@@ -4,5 +4,8 @@
 ToolsTreePackages=
         gcc
         gperf
+        lcov
+        llvm
         meson
         pkgconf
+        rsync

mkosi.conf.d/05-tools/mkosi.conf.d/arch.conf

@@ -10,6 +10,7 @@ ToolsTreePackages=
         libcap
         libmicrohttpd
         mypy
+        perl-json-xs
         python-jinja
         python-pytest
         ruff

mkosi.coverage/mkosi.conf

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Environment=COVERAGE=1
+
+[Content]
+KernelCommandLine=
+        COVERAGE_BUILD_DIR=/coverage
+        systemd.setenv=COVERAGE_BUILD_DIR=/coverage

mkosi.coverage/mkosi.postinst

@@ -0,0 +1,56 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+(
+    shopt -s nullglob
+    rm -f "$BUILDROOT"/coverage/*.gcda
+)
+
+# When using -fprofile-dir=, GCC creates all gcda files under the given directory at the same location as the
+# gcno file in the build directory, but with each '/' replaced with '#'. LLVM creates each gcda file under
+# the given directory without replacing each '/' with '#'. Because we want all processes to be able to write
+# gcda files under /coverage regardless of which user they are running as, we pre-create all files under
+# /coverage and make them world readable and writable so that we don't have to mess with umasks for each
+# process that writes to /coverage.
+if ((LLVM)); then
+    rsync --recursive --include='*/' --exclude='*' --relative "$BUILDDIR" "$BUILDROOT/coverage"
+    find "$BUILDDIR" -name '*.gcno' | sed 's/gcno/gcda/' | xargs -I '{}' touch "$BUILDROOT/coverage/{}"
+else
+    find "$BUILDDIR" -name '*.gcno' | sed 's/gcno/gcda/' | sed 's/\//#/g' | xargs -I '{}' touch "$BUILDROOT/coverage/{}"
+fi
+
+chmod --recursive 777 "$BUILDROOT/coverage"
+
+# When built with gcov, disable ProtectSystem= and ProtectHome= in the test images, since it prevents gcov to
+# write the coverage reports (*.gcda files).
+mkdir -p "$BUILDROOT/usr/lib/systemd/system/service.d/"
+cat >"$BUILDROOT/usr/lib/systemd/system/service.d/99-gcov-override.conf" <<EOF
+[Service]
+ProtectSystem=no
+ProtectHome=no
+EOF
+
+# Similarly, set ReadWritePaths= to the coverage directory in the test image to make the coverage work with
+# units using DynamicUser=yes. Do this only for services with test- prefix and a couple of known-to-use
+# DynamicUser=yes services, as setting this system-wide has many undesirable side-effects, as it creates its
+# own namespace.
+for service in capsule@ test- systemd-journal-{gatewayd,upload}; do
+    mkdir -p "$BUILDROOT/usr/lib/systemd/system/$service.service.d/"
+    cat >"$BUILDROOT/usr/lib/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf" <<EOF
+[Service]
+ReadWritePaths=/coverage
+EOF
+done
+
+# Ditto, but for the user daemon.
+mkdir -p "$BUILDROOT/usr/lib/systemd/user/test-.service.d/"
+cat >"$BUILDROOT/usr/lib/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf" <<EOF
+[Service]
+ReadWritePaths=/coverage
+EOF
+
+# Bind the coverage directory into nspawn containers that are executed using machinectl. Unfortunately, the
+# .nspawn files don't support drop-ins so we have to inject the bind mount directly into the
+# systemd-nspawn@.service unit.
+sed -ri "s/^ExecStart=.+$/& --bind=\/coverage/" "$BUILDROOT/usr/lib/systemd/system/systemd-nspawn@.service"

mkosi.extra.common/usr/lib/systemd/coverage-forwarder

@@ -0,0 +1,9 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+logger --journald <<EOF
+MESSAGE=Tarball with coverage data from /coverage
+COVERAGE_TAR=$(tar --create --file - --directory /coverage --zstd . | base64 --wrap=0)
+EOF
+
+journalctl --flush

mkosi.extra.common/usr/lib/systemd/system-preset/00-mkosi.preset

@@ -39,3 +39,5 @@ disable iscsiuio.socket
 
 # mkosi relabels the image itself so no need to do it on boot.
 disable selinux-autorelabel-mark.service
+
+enable coverage-forwarder.service

mkosi.extra.common/usr/lib/systemd/system/coverage-forwarder.service

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Unit]
+Description=Forward coverage data to the journal before shutting down
+ConditionEnvironment=COVERAGE_BUILD_DIR
+
+DefaultDependencies=no
+After=systemd-journald.socket
+Requires=systemd-journald.socket
+After=shutdown.target initrd-switch-root.target
+Before=final.target initrd-switch-root.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/lib/systemd/coverage-forwarder
+
+[Install]
+WantedBy=final.target initrd-switch-root.target

mkosi.images/build/mkosi.conf.d/arch/mkosi.build.chroot

@@ -32,6 +32,10 @@ MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}"
 if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then
     MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe"
 fi
+if ((COVERAGE)); then
+    MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS -D b_coverage=true"
+    MKOSI_CFLAGS="$MKOSI_CFLAGS -fprofile-dir=/coverage"
+fi
 
 # Override the default options. We specifically disable "strip", "zipman" and "lto" as they slow down builds
 # significantly. OPTIONS= cannot be overridden on the makepkg command line so we append to /etc/makepkg.conf

mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.build.chroot

@@ -52,6 +52,10 @@ MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}"
 if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then
     MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe"
 fi
+if ((COVERAGE)); then
+    MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS -D b_coverage=true"
+    MKOSI_CFLAGS="$MKOSI_CFLAGS -fprofile-dir=/coverage"
+fi
 
 (
     shopt -s nullglob

mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.build.chroot

@@ -48,6 +48,10 @@ MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}"
 if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then
     MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe"
 fi
+if ((COVERAGE)); then
+    MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS -D b_coverage=true"
+    MKOSI_CFLAGS="$MKOSI_CFLAGS -fprofile-dir=/coverage"
+fi
 
 # TODO: Drop GENSYMBOLS_LEVEL once https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=986746 is fixed.
 build() {

mkosi.images/build/mkosi.conf.d/opensuse/mkosi.build.chroot

@@ -52,6 +52,10 @@ MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}"
 if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then
     MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe"
 fi
+if ((COVERAGE)); then
+    MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS -D b_coverage=true"
+    MKOSI_CFLAGS="$MKOSI_CFLAGS -fprofile-dir=/coverage"
+fi
 
 # TODO: Drop when the spec is fixed (either the patch is adapted or not applied when building for upstream).
 sed --in-place '/0009-pid1-handle-console-specificities-weirdness-for-s390.patch/d' "pkg/$PKG_SUBDIR/systemd.spec"

mkosi.images/initrd/mkosi.conf

@@ -4,6 +4,7 @@
 Include=
         mkosi-initrd
         %D/mkosi.sanitizers
+        %D/mkosi.coverage
 
 [Content]
 ExtraTrees=%D/mkosi.extra.common
@@ -12,3 +13,4 @@ Packages=
         findutils
         grep
         sed
+        tar

test/integration-test-wrapper.py

@@ -4,12 +4,15 @@
 """Test wrapper command for driving integration tests."""
 
 import argparse
+import base64
+import dataclasses
 import json
 import os
 import re
 import shlex
 import subprocess
 import sys
+import tempfile
 import textwrap
 from pathlib import Path
 
@@ -33,6 +36,47 @@ ExecStart=false
 """
 
 
+def sandbox(args: argparse.Namespace) -> list[str]:
+    return [
+        args.mkosi,
+        '--directory', os.fspath(args.meson_source_dir),
+        '--extra-search-path', os.fspath(args.meson_build_dir),
+        'sandbox',
+    ]  # fmt: skip
+
+
+@dataclasses.dataclass(frozen=True)
+class Summary:
+    distribution: str
+    release: str
+    architecture: str
+    builddir: Path
+    environment: dict[str, str]
+
+    @classmethod
+    def get(cls, args: argparse.Namespace) -> 'Summary':
+        j = json.loads(
+            subprocess.run(
+                [
+                    args.mkosi,
+                    '--directory', os.fspath(args.meson_source_dir),
+                    '--json',
+                    'summary',
+                ],
+                stdout=subprocess.PIPE,
+                text=True,
+            ).stdout
+        )  # fmt: skip
+
+        return Summary(
+            distribution=j['Images'][-1]['Distribution'],
+            release=j['Images'][-1]['Release'],
+            architecture=j['Images'][-1]['Architecture'],
+            builddir=Path(j['Images'][-1]['BuildDirectory']),
+            environment=j['Images'][-1]['Environment'],
+        )
+
+
 def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
     # Collect executable paths of all coredumps and filter out the expected ones.
 
@@ -42,11 +86,7 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
         exclude_regex = None
 
     result = subprocess.run(
-        [
-            args.mkosi,
-            '--directory', os.fspath(args.meson_source_dir),
-            '--extra-search-path', os.fspath(args.meson_build_dir),
-            'sandbox',
+        sandbox(args) + [
             'coredumpctl',
             '--file', journal_file,
             '--json=short',
@@ -69,11 +109,7 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
         return False
 
     subprocess.run(
-        [
-            args.mkosi,
-            '--directory', os.fspath(args.meson_source_dir),
-            '--extra-search-path', os.fspath(args.meson_build_dir),
-            'sandbox',
+        sandbox(args) + [
             'coredumpctl',
             '--file', journal_file,
             '--no-pager',
@@ -86,6 +122,119 @@ def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
     return True
 
 
+def process_coverage(args: argparse.Namespace, summary: Summary, name: str, journal_file: Path) -> None:
+    coverage = subprocess.run(
+        sandbox(args) + [
+            'journalctl',
+            '--file', journal_file,
+            '--field=COVERAGE_TAR',
+        ],
+        stdout=subprocess.PIPE,
+        text=True,
+        check=True,
+    ).stdout  # fmt: skip
+
+    (args.meson_build_dir / 'test/coverage').mkdir(exist_ok=True)
+
+    initial = args.meson_build_dir / 'test/coverage/initial.coverage-info'
+    output = args.meson_build_dir / f'test/coverage/{name}.coverage-info'
+
+    for b64 in coverage.splitlines():
+        tarball = base64.b64decode(b64)
+
+        with tempfile.TemporaryDirectory(prefix='coverage-') as tmp:
+            subprocess.run(
+                sandbox(args) + [
+                    'tar',
+                    '--extract',
+                    '--file', '-',
+                    '--directory', tmp,
+                    '--keep-directory-symlink',
+                    '--no-overwrite-dir',
+                    '--zstd',
+                ],
+                input=tarball,
+                check=True,
+            )  # fmt: skip
+
+            for p in Path(tmp).iterdir():
+                if not p.name.startswith('#'):
+                    continue
+
+                dst = Path(tmp) / p.name.replace('#', '/').lstrip('/')
+                dst.parent.mkdir(parents=True, exist_ok=True)
+                p.rename(dst)
+
+            subprocess.run(
+                sandbox(args) + [
+                    'find',
+                    tmp,
+                    '-name', '*.gcda',
+                    '-size', '0',
+                    '-delete',
+                ],
+                input=tarball,
+                check=True,
+            )  # fmt: skip
+
+            subprocess.run(
+                sandbox(args)
+                + [
+                    'rsync',
+                    '--archive',
+                    '--prune-empty-dirs',
+                    '--include=*/',
+                    '--include=*.gcno',
+                    '--exclude=*',
+                    f'{os.fspath(args.meson_build_dir / summary.builddir)}/',
+                    os.fspath(Path(tmp) / 'work/build'),
+                ],
+                check=True,
+            )
+
+            subprocess.run(
+                sandbox(args)
+                + [
+                    'lcov',
+                    *(
+                        [
+                            '--gcov-tool', 'llvm-cov',
+                            '--gcov-tool', 'gcov',
+                        ]
+                        if summary.environment.get('LLVM', '0') == '1'
+                        else []
+                    ),
+                    '--directory', tmp,
+                    '--base-directory', 'src/',
+                    '--capture',
+                    '--exclude', '*.gperf',
+                    '--output-file', f'{output}.new',
+                    '--ignore-errors', 'inconsistent,inconsistent,source,negative',
+                    '--substitute', 's#src/src#src#g',
+                    '--no-external',
+                    '--quiet',
+                ],
+                check=True,
+            )  # fmt: skip
+
+            subprocess.run(
+                sandbox(args)
+                + [
+                    'lcov',
+                    '--ignore-errors', 'inconsistent,inconsistent,format,corrupt,empty',
+                    '--add-tracefile', output if output.exists() else initial,
+                    '--add-tracefile', f'{output}.new',
+                    '--output-file', output,
+                    '--quiet',
+                ],
+                check=True,
+            )  # fmt: skip
+
+            Path(f'{output}.new').unlink()
+
+            print(f'Wrote coverage report for {name} to {output}', file=sys.stderr)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--mkosi', required=True)
@@ -127,6 +276,7 @@ def main() -> None:
 
     keep_journal = os.getenv('TEST_SAVE_JOURNAL', 'fail')
     shell = bool(int(os.getenv('TEST_SHELL', '0')))
+    summary = Summary.get(args)
 
     if shell and not sys.stderr.isatty():
         print(
@@ -250,6 +400,13 @@ def main() -> None:
 
     coredumps = process_coredumps(args, journal_file)
 
+    if (
+        summary.environment.get('COVERAGE', '0') == '1'
+        and result.returncode in (args.exit_code, 77)
+        and not coredumps
+    ):
+        process_coverage(args, summary, name, journal_file)
+
     if keep_journal == '0' or (
         keep_journal == 'fail' and result.returncode in (args.exit_code, 77) and not coredumps
     ):
@@ -262,22 +419,11 @@ def main() -> None:
 
     if os.getenv('GITHUB_ACTIONS'):
         id = os.environ['GITHUB_RUN_ID']
+        workflow = os.environ['GITHUB_WORKFLOW']
         iteration = os.environ['GITHUB_RUN_ATTEMPT']
-        j = json.loads(
-            subprocess.run(
-                [
-                    args.mkosi,
-                    '--directory', os.fspath(args.meson_source_dir),
-                    '--json',
-                    'summary',
-                ],
-                stdout=subprocess.PIPE,
-                text=True,
-            ).stdout
-        )  # fmt: skip
-        distribution = j['Images'][-1]['Distribution']
-        release = j['Images'][-1]['Release']
-        artifact = f'ci-mkosi-{id}-{iteration}-{distribution}-{release}-failed-test-journals'
+        artifact = (
+            f'ci-{workflow}-{id}-{iteration}-{summary.distribution}-{summary.release}-failed-test-journals'
+        )
         ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}']
         journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal')
 

test/test-network/systemd-networkd-tests.py

@@ -8655,7 +8655,7 @@ if __name__ == '__main__':
     asan_options = ns.asan_options
     lsan_options = ns.lsan_options
     ubsan_options = ns.ubsan_options
-    with_coverage = ns.with_coverage
+    with_coverage = ns.with_coverage or "COVERAGE_BUILD_DIR" in os.environ
     show_journal = ns.show_journal
 
     if use_valgrind: