mkosi: Sanitizer improvements

- Let's set the environment on the kernel command line so it applies
to initrd and main system.
- Let's add the necessary wrappers that are also added in test-functions.
Unlike test-functions we don't use gcc/clang to get the library path as
that requires installing gcc/clang in the initrd.
- Let's drop the hack to get journald writing to the console and have
it write to kmsg instead. We'll get the output either way.
- Stop removing libstdc++ and sanitizer libraries from Arch Linux
initrds and other images as it's required by the sanitizer libraries.
- Add a workaround for specifying extra meson options for opensuse
- Add a leak sanitizer suppression file as a workaround for a false
positive leak in verify_selinuxmnt() in libselinux. We do a soname match
because the stacktrace can't be properly symbolized on Debian.

.github/workflows/mkosi.yml

@@ -119,6 +119,7 @@ jobs:
         [Host]
         ToolsTree=default
         ToolsTreeDistribution=fedora
+        QemuMem=4G
         # We build with debuginfo so there's no point in mounting the sources into the machine.
         RuntimeBuildSources=no
         EOF

mkosi.conf

@@ -10,13 +10,9 @@ MinimumVersion=23~devel
 @CacheDirectory=build/mkosi.cache
 
 [Content]
-# Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_.
-Environment=ASAN_OPTIONS=verify_asan_link_order=false
-            MKOSI_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
-            MKOSI_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
-            # The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later
-            # versions of mkfs.ext4 enabled it by default, so we disable it explicitly.
-            SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file"
+# The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later
+# versions of mkfs.ext4 enabled it by default, so we disable it explicitly.
+Environment=SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file"
 @SELinuxRelabel=no
 BuildSourcesEphemeral=yes
 

mkosi.conf.d/20-sanitizers.conf

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+Environment=SANITIZERS
+
+[Content]
+# Set verify_asan_link_order=0 to prevent ASAN warnings when building the image and make sure the real ASAN
+# options are set when booting the image.
+# Set intercept_tls_get_addr=0 to work around leak sanitizer segmentation fault in test-dlopen-so on CentOS
+# Stream 9.
+# TODO: Drop intercept_tls_get_addr=0 when we remove CentOS Stream 9 builds.
+Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0
+KernelCommandLine=
+        ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
+        systemd.setenv=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
+        UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
+        systemd.setenv=UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
+        LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions
+        systemd.setenv=LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions

mkosi.images/exitrd/mkosi.conf.d/10-arch.conf

@@ -15,11 +15,6 @@ RemoveFiles=
         /usr/lib/libgomp.so*
         /usr/lib/libgphobos.so*
         /usr/lib/libobjc.so*
-        /usr/lib/libasan.so*
-        /usr/lib/libtsan.so*
-        /usr/lib/liblsan.so*
-        /usr/lib/libubsan.so*
-        /usr/lib/libstdc++.so*
         /usr/lib/libgdruntime.so*
 
         # Remove all files that are only required for development.

mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf

@@ -17,11 +17,6 @@ RemoveFiles=
         /usr/lib/libgomp.so*
         /usr/lib/libgphobos.so*
         /usr/lib/libobjc.so*
-        /usr/lib/libasan.so*
-        /usr/lib/libtsan.so*
-        /usr/lib/liblsan.so*
-        /usr/lib/libubsan.so*
-        /usr/lib/libstdc++.so*
         /usr/lib/libgdruntime.so*
 
         # Remove all files that are only required for development.

mkosi.images/system/initrd/mkosi.conf

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Content]
+PostInstallationScripts=../mkosi.sanitizers.chroot
+ExtraTrees=../leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions

mkosi.images/system/leak-sanitizer-suppressions

@@ -0,0 +1 @@
+leak:libselinux

mkosi.images/system/mkosi.conf

@@ -26,6 +26,14 @@ ExtraTrees=
         %O/minimal-1.root-%a-verity-sig.raw:/usr/share/minimal_1.verity.sig
         %O/minimal-base:/usr/share/TEST-13-NSPAWN-container-template
         %O/exitrd:/exitrd
+        leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions
+
+PostInstallationScripts=mkosi.sanitizers.chroot
+
+InitrdPackages=
+        findutils
+        grep
+        sed
 
 Packages=
         acl

mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot

@@ -51,6 +51,8 @@ build() {
     IFS=
     # TODO: Replace meson_build and meson_install overrides with "--undefine __meson_verbose" once
     # https://github.com/mesonbuild/meson/pull/12835 is available.
+    # TODO: Replace __meson_auto_features override with meson_extra_configure_options once the suse spec
+    # starts to use it.
     # shellcheck disable=SC2046
     rpmbuild \
         -bb \
@@ -71,7 +73,7 @@ build() {
         --define "build_cflags $(rpm --eval %build_cflags) $EXTRA_CFLAGS" \
         --define "meson_build %{shrink:%{__meson} compile -C %{_vpath_builddir} -j %{_smp_build_ncpus} %{nil}}" \
         --define "meson_install %{shrink:DESTDIR=%{buildroot} %{__meson} install -C %{_vpath_builddir} --no-rebuild --quiet %{nil}}" \
-        --define "meson_extra_configure_options -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \
+        --define "__meson_auto_features auto -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \
         --define "__os_install_post /usr/lib/rpm/brp-suse %{nil}" \
         --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \
         --define "__script_requires %{nil}" \

mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+# The iscsi-init.service calls `sh` which might, in certain circumstances, pull in instrumented systemd NSS
+# modules causing `sh` to fail. Avoid the issue by setting LD_PRELOAD to load the sanitizer libraries if
+# needed.
+[Service]
+EnvironmentFile=-/usr/lib/systemd/systemd-asan-env

mkosi.images/system/mkosi.postinst.chroot

@@ -2,48 +2,6 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -e
 
-if [ -n "$SANITIZERS" ]; then
-    LD_PRELOAD=$(ldd /usr/lib/systemd/systemd | grep libasan.so | awk '{print $3}')
-
-    mkdir -p /etc/systemd/system.conf.d
-
-    cat >/etc/systemd/system.conf.d/10-asan.conf <<EOF
-[Manager]
-ManagerEnvironment=ASAN_OPTIONS=$MKOSI_ASAN_OPTIONS\\
-                   UBSAN_OPTIONS=$MKOSI_UBSAN_OPTIONS\\
-                   LD_PRELOAD=$LD_PRELOAD
-DefaultEnvironment=ASAN_OPTIONS=$MKOSI_ASAN_OPTIONS\\
-                   UBSAN_OPTIONS=$MKOSI_UBSAN_OPTIONS\\
-                   LD_PRELOAD=$LD_PRELOAD
-EOF
-
-    # ASAN logs to stderr by default. However, journald's stderr is connected to /dev/null, so we lose
-    # all the ASAN logs. To rectify that, let's connect journald's stdout to the console so that any
-    # sanitizer failures appear directly on the user's console.
-    mkdir -p /etc/systemd/system/systemd-journald.service.d
-    cat >/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf <<EOF
-[Service]
-StandardOutput=tty
-EOF
-
-    # Both systemd and util-linux's login call vhangup() on /dev/console which disconnects all users.
-    # This means systemd-journald can't log to /dev/console even if we configure `StandardOutput=tty`. As
-    # a workaround, we modify console-getty.service to disable systemd's vhangup() and disallow login
-    # from calling vhangup() so that journald's ASAN logs correctly end up in the console.
-
-    mkdir -p /etc/systemd/system/console-getty.service.d
-    cat >/etc/systemd/system/console-getty.service.d/10-no-vhangup.conf <<EOF
-[Service]
-TTYVHangup=no
-CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
-EOF
-    # ASAN and syscall filters aren't compatible with each other.
-    find /usr /etc -name '*.service' -type f -exec sed -i 's/^\(MemoryDeny\|SystemCall\)/# \1/' {} +
-
-    # `systemd-hwdb update` takes > 50s when built with sanitizers so let's not run it by default.
-    systemctl mask systemd-hwdb-update.service
-fi
-
 if command -v authselect >/dev/null; then
     # authselect 1.5.0 renamed the minimal profile to the local profile without keeping backwards compat so
     # let's use the new name if it exists.

mkosi.images/system/mkosi.sanitizers.chroot

@@ -0,0 +1,130 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+if [[ -z "$SANITIZERS" ]]; then
+    exit 0
+fi
+
+# Sanitizers log to stderr by default. However, journald's stderr is connected to /dev/null, so we lose
+# all the sanitizer logs. To rectify that, let's connect journald's stdout to kmsg so that the sanitizer
+# failures end up in the journal.
+mkdir -p /etc/systemd/system/systemd-journald.service.d
+cat >/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf <<EOF
+[Service]
+StandardOutput=kmsg
+EOF
+
+# ASAN and syscall filters aren't compatible with each other.
+find /usr /etc -name '*.service' -type f -exec sed -i 's/^\(MemoryDeny\|SystemCall\)/# \1/' {} +
+
+# `systemd-hwdb update` takes > 50s when built with sanitizers so let's not run it by default.
+systemctl mask systemd-hwdb-update.service
+
+ASAN_RT_PATH="$(grep libasan.so < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)"
+if [[ -z "$ASAN_RT_PATH" ]]; then
+    ASAN_RT_PATH="$(grep libclang_rt.asan < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)"
+
+    # As clang's ASan DSO is usually in a non-standard path, let's check if
+    # the environment is set accordingly. If not, warn the user and exit.
+    # We're not setting the LD_LIBRARY_PATH automagically here, because
+    # user should encounter (and fix) the same issue when running the unit
+    # tests (meson test)
+    if ldd /usr/lib/systemd/systemd | grep -q "libclang_rt.asan.*not found"; then
+        echo >&2 "clang's ASan DSO libclang_rt.asan is not present in the runtime library path"
+        exit 1
+    fi
+fi
+if [[ -z "$ASAN_RT_PATH" ]]; then
+    echo >&2 "systemd is not linked against the ASan DSO"
+    echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
+    exit 1
+fi
+
+wrap=(
+    /usr/lib/polkit-1/polkitd
+    /usr/libexec/polkit-1/polkitd
+    agetty
+    btrfs
+    capsh
+    chgrp
+    chown
+    cryptsetup
+    curl
+    dbus-broker-launch
+    dbus-daemon
+    delv
+    dhcpd
+    dig
+    dmsetup
+    dnsmasq
+    findmnt
+    getent
+    getfacl
+    id
+    integritysetup
+    iscsid
+    kpartx
+    logger
+    login
+    ls
+    lsblk
+    lvm
+    mdadm
+    mkfs.btrfs
+    mkfs.erofs
+    mkfs.ext4
+    mkfs.vfat
+    mkfs.xfs
+    mksquashfs
+    mkswap
+    multipath
+    multipathd
+    nvme
+    p11-kit
+    pkill
+    ps
+    setfacl
+    setpriv
+    sshd
+    stat
+    su
+    tar
+    tgtd
+    useradd
+    userdel
+    veritysetup
+)
+
+for bin in "${wrap[@]}"; do
+    if ! command -v "$bin" >/dev/null; then
+        continue
+    fi
+
+    if [[ "$bin" == getent ]]; then
+        enable_lsan=1
+    else
+        enable_lsan=0
+    fi
+
+    target="$(command -v "$bin")"
+
+    mv "$target" "$target.orig"
+
+    cat >"$target" <<EOF
+#!/bin/bash
+# Preload the ASan runtime DSO, otherwise ASAn will complain
+export LD_PRELOAD="$ASAN_RT_PATH"
+# Disable LSan to speed things up, since we don't care about leak reports
+# from 'external' binaries
+export ASAN_OPTIONS=detect_leaks=$enable_lsan
+# Set argv[0] to the original binary name without the ".orig" suffix
+exec -a "\$0" -- "${target}.orig" "\$@"
+EOF
+    chmod +x "$target"
+done
+
+cat >/usr/lib/systemd/systemd-asan-env <<EOF
+LD_PRELOAD=$ASAN_RT_PATH
+LSAN_OPTIONS=detect_leaks=0
+EOF