From 8919f86f573f5256283298415027b0a9052507e5 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 16 May 2024 17:18:38 +0200 Subject: [PATCH] mkosi: Sanitizer improvements - Let's set the environment on the kernel command line so it applies to initrd and main system. - Let's add the necessary wrappers that are also added in test-functions. Unlike test-functions we don't use gcc/clang to get the library path as that requires installing gcc/clang in the initrd. - Let's drop the hack to get journald writing to the console and have it write to kmsg instead. We'll get the output either way. - Stop removing libstdc++ and sanitizer libraries from Arch Linux initrds and other images as it's required by the sanitizer libraries. - Add a workaround for specifying extra meson options for opensuse - Add a leak sanitizer suppression file as a workaround for a false positive leak in verify_selinuxmnt() in libselinux. We do a soname match because the stacktrace can't be properly symbolized on Debian. --- .github/workflows/mkosi.yml | 1 + mkosi.conf | 10 +- mkosi.conf.d/20-sanitizers.conf | 19 +++ mkosi.images/exitrd/mkosi.conf.d/10-arch.conf | 5 - .../minimal-base/mkosi.conf.d/10-arch.conf | 5 - mkosi.images/system/initrd/mkosi.conf | 5 + .../system/leak-sanitizer-suppressions | 1 + mkosi.images/system/mkosi.conf | 8 ++ .../10-opensuse/mkosi.build.chroot | 4 +- .../system/iscsi-init.service.d/asan.conf | 7 + mkosi.images/system/mkosi.postinst.chroot | 42 ------ mkosi.images/system/mkosi.sanitizers.chroot | 130 ++++++++++++++++++ 12 files changed, 177 insertions(+), 60 deletions(-) create mode 100644 mkosi.conf.d/20-sanitizers.conf create mode 100644 mkosi.images/system/initrd/mkosi.conf create mode 100644 mkosi.images/system/leak-sanitizer-suppressions create mode 100644 mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf create mode 100755 mkosi.images/system/mkosi.sanitizers.chroot diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 32dab1a7f3a..583f287de29 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -119,6 +119,7 @@ jobs: [Host] ToolsTree=default ToolsTreeDistribution=fedora + QemuMem=4G # We build with debuginfo so there's no point in mounting the sources into the machine. RuntimeBuildSources=no EOF diff --git a/mkosi.conf b/mkosi.conf index 300b86bf978..1c552a269e0 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -10,13 +10,9 @@ MinimumVersion=23~devel @CacheDirectory=build/mkosi.cache [Content] -# Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_. -Environment=ASAN_OPTIONS=verify_asan_link_order=false - MKOSI_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 - MKOSI_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 - # The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later - # versions of mkfs.ext4 enabled it by default, so we disable it explicitly. - SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file" +# The kernel versions in CentOS Stream 9 and Ubuntu 22.04 don't support orphan_file, but later +# versions of mkfs.ext4 enabled it by default, so we disable it explicitly. +Environment=SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file" @SELinuxRelabel=no BuildSourcesEphemeral=yes diff --git a/mkosi.conf.d/20-sanitizers.conf b/mkosi.conf.d/20-sanitizers.conf new file mode 100644 index 00000000000..235b233e1ad --- /dev/null +++ b/mkosi.conf.d/20-sanitizers.conf @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Match] +Environment=SANITIZERS + +[Content] +# Set verify_asan_link_order=0 to prevent ASAN warnings when building the image and make sure the real ASAN +# options are set when booting the image. +# Set intercept_tls_get_addr=0 to work around leak sanitizer segmentation fault in test-dlopen-so on CentOS +# Stream 9. +# TODO: Drop intercept_tls_get_addr=0 when we remove CentOS Stream 9 builds. +Environment=ASAN_OPTIONS=verify_asan_link_order=0:intercept_tls_get_addr=0 +KernelCommandLine= + ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + systemd.setenv=ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1 + UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 + systemd.setenv=UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 + LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions + systemd.setenv=LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf b/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf index 25d20887ffa..c8b1904f6f4 100644 --- a/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf +++ b/mkosi.images/exitrd/mkosi.conf.d/10-arch.conf @@ -15,11 +15,6 @@ RemoveFiles= /usr/lib/libgomp.so* /usr/lib/libgphobos.so* /usr/lib/libobjc.so* - /usr/lib/libasan.so* - /usr/lib/libtsan.so* - /usr/lib/liblsan.so* - /usr/lib/libubsan.so* - /usr/lib/libstdc++.so* /usr/lib/libgdruntime.so* # Remove all files that are only required for development. diff --git a/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf b/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf index 30e8fda59e0..9b033975d64 100644 --- a/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf +++ b/mkosi.images/minimal-base/mkosi.conf.d/10-arch.conf @@ -17,11 +17,6 @@ RemoveFiles= /usr/lib/libgomp.so* /usr/lib/libgphobos.so* /usr/lib/libobjc.so* - /usr/lib/libasan.so* - /usr/lib/libtsan.so* - /usr/lib/liblsan.so* - /usr/lib/libubsan.so* - /usr/lib/libstdc++.so* /usr/lib/libgdruntime.so* # Remove all files that are only required for development. diff --git a/mkosi.images/system/initrd/mkosi.conf b/mkosi.images/system/initrd/mkosi.conf new file mode 100644 index 00000000000..56bd4d0aa7c --- /dev/null +++ b/mkosi.images/system/initrd/mkosi.conf @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +[Content] +PostInstallationScripts=../mkosi.sanitizers.chroot +ExtraTrees=../leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions diff --git a/mkosi.images/system/leak-sanitizer-suppressions b/mkosi.images/system/leak-sanitizer-suppressions new file mode 100644 index 00000000000..639abb8f3f9 --- /dev/null +++ b/mkosi.images/system/leak-sanitizer-suppressions @@ -0,0 +1 @@ +leak:libselinux diff --git a/mkosi.images/system/mkosi.conf b/mkosi.images/system/mkosi.conf index 6455b0477e2..5d33cba7ee8 100644 --- a/mkosi.images/system/mkosi.conf +++ b/mkosi.images/system/mkosi.conf @@ -26,6 +26,14 @@ ExtraTrees= %O/minimal-1.root-%a-verity-sig.raw:/usr/share/minimal_1.verity.sig %O/minimal-base:/usr/share/TEST-13-NSPAWN-container-template %O/exitrd:/exitrd + leak-sanitizer-suppressions:/usr/lib/systemd/leak-sanitizer-suppressions + +PostInstallationScripts=mkosi.sanitizers.chroot + +InitrdPackages= + findutils + grep + sed Packages= acl diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index b2c56fda776..13cda4c9b89 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -51,6 +51,8 @@ build() { IFS= # TODO: Replace meson_build and meson_install overrides with "--undefine __meson_verbose" once # https://github.com/mesonbuild/meson/pull/12835 is available. + # TODO: Replace __meson_auto_features override with meson_extra_configure_options once the suse spec + # starts to use it. # shellcheck disable=SC2046 rpmbuild \ -bb \ @@ -71,7 +73,7 @@ build() { --define "build_cflags $(rpm --eval %build_cflags) $EXTRA_CFLAGS" \ --define "meson_build %{shrink:%{__meson} compile -C %{_vpath_builddir} -j %{_smp_build_ncpus} %{nil}}" \ --define "meson_install %{shrink:DESTDIR=%{buildroot} %{__meson} install -C %{_vpath_builddir} --no-rebuild --quiet %{nil}}" \ - --define "meson_extra_configure_options -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \ + --define "__meson_auto_features auto -D mode=developer -D b_sanitize=${SANITIZERS:-none}" \ --define "__os_install_post /usr/lib/rpm/brp-suse %{nil}" \ --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ diff --git a/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf b/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf new file mode 100644 index 00000000000..ebf7899a789 --- /dev/null +++ b/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/iscsi-init.service.d/asan.conf @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +# The iscsi-init.service calls `sh` which might, in certain circumstances, pull in instrumented systemd NSS +# modules causing `sh` to fail. Avoid the issue by setting LD_PRELOAD to load the sanitizer libraries if +# needed. +[Service] +EnvironmentFile=-/usr/lib/systemd/systemd-asan-env diff --git a/mkosi.images/system/mkosi.postinst.chroot b/mkosi.images/system/mkosi.postinst.chroot index 15f268a20ae..397884b7202 100755 --- a/mkosi.images/system/mkosi.postinst.chroot +++ b/mkosi.images/system/mkosi.postinst.chroot @@ -2,48 +2,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ -n "$SANITIZERS" ]; then - LD_PRELOAD=$(ldd /usr/lib/systemd/systemd | grep libasan.so | awk '{print $3}') - - mkdir -p /etc/systemd/system.conf.d - - cat >/etc/systemd/system.conf.d/10-asan.conf </etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf </etc/systemd/system/console-getty.service.d/10-no-vhangup.conf < 50s when built with sanitizers so let's not run it by default. - systemctl mask systemd-hwdb-update.service -fi - if command -v authselect >/dev/null; then # authselect 1.5.0 renamed the minimal profile to the local profile without keeping backwards compat so # let's use the new name if it exists. diff --git a/mkosi.images/system/mkosi.sanitizers.chroot b/mkosi.images/system/mkosi.sanitizers.chroot new file mode 100755 index 00000000000..48c5d147aa5 --- /dev/null +++ b/mkosi.images/system/mkosi.sanitizers.chroot @@ -0,0 +1,130 @@ +#!/bin/bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -e + +if [[ -z "$SANITIZERS" ]]; then + exit 0 +fi + +# Sanitizers log to stderr by default. However, journald's stderr is connected to /dev/null, so we lose +# all the sanitizer logs. To rectify that, let's connect journald's stdout to kmsg so that the sanitizer +# failures end up in the journal. +mkdir -p /etc/systemd/system/systemd-journald.service.d +cat >/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf < 50s when built with sanitizers so let's not run it by default. +systemctl mask systemd-hwdb-update.service + +ASAN_RT_PATH="$(grep libasan.so < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)" +if [[ -z "$ASAN_RT_PATH" ]]; then + ASAN_RT_PATH="$(grep libclang_rt.asan < <(ldd /usr/lib/systemd/systemd) | cut -d ' ' -f 3)" + + # As clang's ASan DSO is usually in a non-standard path, let's check if + # the environment is set accordingly. If not, warn the user and exit. + # We're not setting the LD_LIBRARY_PATH automagically here, because + # user should encounter (and fix) the same issue when running the unit + # tests (meson test) + if ldd /usr/lib/systemd/systemd | grep -q "libclang_rt.asan.*not found"; then + echo >&2 "clang's ASan DSO libclang_rt.asan is not present in the runtime library path" + exit 1 + fi +fi +if [[ -z "$ASAN_RT_PATH" ]]; then + echo >&2 "systemd is not linked against the ASan DSO" + echo >&2 "gcc does this by default, for clang compile with -shared-libasan" + exit 1 +fi + +wrap=( + /usr/lib/polkit-1/polkitd + /usr/libexec/polkit-1/polkitd + agetty + btrfs + capsh + chgrp + chown + cryptsetup + curl + dbus-broker-launch + dbus-daemon + delv + dhcpd + dig + dmsetup + dnsmasq + findmnt + getent + getfacl + id + integritysetup + iscsid + kpartx + logger + login + ls + lsblk + lvm + mdadm + mkfs.btrfs + mkfs.erofs + mkfs.ext4 + mkfs.vfat + mkfs.xfs + mksquashfs + mkswap + multipath + multipathd + nvme + p11-kit + pkill + ps + setfacl + setpriv + sshd + stat + su + tar + tgtd + useradd + userdel + veritysetup +) + +for bin in "${wrap[@]}"; do + if ! command -v "$bin" >/dev/null; then + continue + fi + + if [[ "$bin" == getent ]]; then + enable_lsan=1 + else + enable_lsan=0 + fi + + target="$(command -v "$bin")" + + mv "$target" "$target.orig" + + cat >"$target" </usr/lib/systemd/systemd-asan-env <