Merge pull request #11252 from evverx/use-asan-wrapper-on-travis-ci

travis: run PID1, journald and everything else under ASan+UBsan

.travis.yml

@@ -1,4 +1,5 @@
 sudo: required
+dist: xenial
 services:
     - docker
 
@@ -17,80 +18,7 @@ stages:
 jobs:
     include:
         - stage: Build & test
-          name: Fedora Latest
-          language: bash
-          env:
-              - FEDORA_RELEASE="latest"
-              - CONT_NAME="systemd-fedora-$FEDORA_RELEASE"
-              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/fedora.sh SETUP
-          script:
-              - set -e
-              # Build systemd
-              - $CI_MANAGERS/fedora.sh RUN
-              - set +e
-          after_script:
-              - $CI_MANAGERS/fedora.sh CLEANUP
-
-        - name: Fedora Latest (ASan+UBSan)
-          language: bash
-          env:
-              - FEDORA_RELEASE="latest"
-              - CONT_NAME="systemd-fedora-$FEDORA_RELEASE"
-              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/fedora.sh SETUP
-          script:
-              - set -e
-              - $CI_MANAGERS/fedora.sh RUN_ASAN
-              - set +e
-          after_script:
-              - $CI_MANAGERS/fedora.sh CLEANUP
-
-        - name: Fedora Latest (clang)
-          language: bash
-          env:
-              - FEDORA_RELEASE="latest"
-              - CONT_NAME="systemd-fedora-$FEDORA_RELEASE"
-              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/fedora.sh SETUP
-          script:
-              - set -e
-              - $CI_MANAGERS/fedora.sh RUN_CLANG
-              - set +e
-          after_script:
-              - $CI_MANAGERS/fedora.sh CLEANUP
-
-        - name: Fedora Latest (clang ASan+UBSan)
-          language: bash
-          env:
-              - FEDORA_RELEASE="latest"
-              - CONT_NAME="systemd-fedora-$FEDORA_RELEASE"
-              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
-          before_install:
-              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-              - docker --version
-          install:
-              - $CI_MANAGERS/fedora.sh SETUP
-          script:
-              - set -e
-              - $CI_MANAGERS/fedora.sh RUN_CLANG_ASAN
-              - set +e
-          after_script:
-              - $CI_MANAGERS/fedora.sh CLEANUP
-
-        - name: Debian Testing
+          name: Debian Testing
           language: bash
           env:
               - DEBIAN_RELEASE="testing"
@@ -103,11 +31,73 @@ jobs:
               - $CI_MANAGERS/debian.sh SETUP
           script:
               - set -e
+              # Build systemd
               - $CI_MANAGERS/debian.sh RUN
               - set +e
           after_script:
               - $CI_MANAGERS/debian.sh CLEANUP
 
+        - name: Debian Testing (ASan+UBSan)
+          language: bash
+          env:
+              - DEBIAN_RELEASE="testing"
+              - CONT_NAME="systemd-debian-$DEBIAN_RELEASE"
+              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
+          before_install:
+              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+              - docker --version
+          install:
+              - $CI_MANAGERS/debian.sh SETUP
+          script:
+              - set -e
+              - $CI_MANAGERS/debian.sh RUN_ASAN
+              - set +e
+          after_script:
+              - $CI_MANAGERS/debian.sh CLEANUP
+
+        - name: Debian Testing (clang)
+          language: bash
+          env:
+              - DEBIAN_RELEASE="testing"
+              - CONT_NAME="systemd-debian-$DEBIAN_RELEASE"
+              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
+          before_install:
+              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+              - docker --version
+          install:
+              - $CI_MANAGERS/debian.sh SETUP
+          script:
+              - set -e
+              - $CI_MANAGERS/debian.sh RUN_CLANG
+              - set +e
+          after_script:
+              - $CI_MANAGERS/debian.sh CLEANUP
+
+        - name: Debian Testing (clang ASan+UBSan)
+          language: bash
+          env:
+              - DEBIAN_RELEASE="testing"
+              - CONT_NAME="systemd-debian-$DEBIAN_RELEASE"
+              - DOCKER_EXEC="docker exec -ti $CONT_NAME"
+          before_install:
+              - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+              - docker --version
+          install:
+              - $CI_MANAGERS/debian.sh SETUP
+          script:
+              - set -e
+              - $CI_MANAGERS/debian.sh RUN_CLANG_ASAN
+              - set +e
+          after_script:
+              - $CI_MANAGERS/debian.sh CLEANUP
+
+        - name: Ubuntu Xenial
+          language: bash
+          script:
+              - set -e
+              - sudo $CI_MANAGERS/xenial.sh
+              - set +e
+
         - stage: Coverity
           language: bash
           env:

src/core/main.c

@@ -83,6 +83,10 @@
 #include "virt.h"
 #include "watchdog.h"
 
+#if HAS_FEATURE_ADDRESS_SANITIZER
+#include <sanitizer/lsan_interface.h>
+#endif
+
 static enum {
         ACTION_RUN,
         ACTION_HELP,
@@ -2612,6 +2616,10 @@ finish:
         }
 #endif
 
+#if HAS_FEATURE_ADDRESS_SANITIZER
+        __lsan_do_leak_check();
+#endif
+
         if (shutdown_verb) {
                 r = become_shutdown(shutdown_verb, retval);
                 log_error_errno(r, "Failed to execute shutdown binary, %s: %m", getpid_cached() == 1 ? "freezing" : "quitting");

src/journal/journald-context.c

@@ -246,7 +246,7 @@ static int client_context_read_label(
 }
 
 static int client_context_read_cgroup(Server *s, ClientContext *c, const char *unit_id) {
-        char *t = NULL;
+        _cleanup_free_ char *t = NULL;
         int r;
 
         assert(c);
@@ -254,7 +254,6 @@ static int client_context_read_cgroup(Server *s, ClientContext *c, const char *u
         /* Try to acquire the current cgroup path */
         r = cg_pid_get_path_shifted(c->pid, s->cgroup_root, &t);
         if (r < 0 || empty_or_root(t)) {
-
                 /* We use the unit ID passed in as fallback if we have nothing cached yet and cg_pid_get_path_shifted()
                  * failed or process is running in a root cgroup. Zombie processes are automatically migrated to root cgroup
                  * on cgroup v1 and we want to be able to map log messages from them too. */
@@ -268,10 +267,8 @@ static int client_context_read_cgroup(Server *s, ClientContext *c, const char *u
         }
 
         /* Let's shortcut this if the cgroup path didn't change */
-        if (streq_ptr(c->cgroup, t)) {
-                free(t);
+        if (streq_ptr(c->cgroup, t))
                 return 0;
-        }
 
         free_and_replace(c->cgroup, t);
 

src/test/test-bpf.c

@@ -30,6 +30,9 @@ int main(int argc, char *argv[]) {
 
         test_setup_logging(LOG_DEBUG);
 
+        if (is_run_on_travis_ci())
+                return log_tests_skipped("test-bpf fails on Travis CI: https://github.com/systemd/systemd/issues/9666");
+
         r = enter_cgroup_subroot();
         if (r == -ENOMEDIUM)
                 return log_tests_skipped("cgroupfs not available");

test/TEST-01-BASIC/test.sh

@@ -3,7 +3,7 @@
 # ex: ts=8 sw=4 sts=4 et filetype=sh
 set -e
 TEST_DESCRIPTION="Basic systemd setup"
-RUN_IN_UNPRIVILEGED_CONTAINER=yes
+RUN_IN_UNPRIVILEGED_CONTAINER=${RUN_IN_UNPRIVILEGED_CONTAINER:-yes}
 
 . $TEST_BASE_DIR/test-functions
 
@@ -26,7 +26,7 @@ Description=Testsuite service
 After=multi-user.target
 
 [Service]
-ExecStart=/bin/sh -x -c 'systemctl --state=failed --no-legend --no-pager > /failed ; echo OK > /testok'
+ExecStart=/bin/sh -x -c 'systemctl --state=failed --no-legend --no-pager > /failed ; systemctl daemon-reload ; echo OK > /testok'
 Type=oneshot
 EOF
 

test/test-functions

@@ -37,7 +37,7 @@ is_built_with_asan() {
     fi
 
     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
-    local _asan_calls=$(objdump -dC $BUILD_DIR/systemd | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
+    local _asan_calls=$(objdump -dC $BUILD_DIR/systemd-journald | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
     if (( $_asan_calls < 1000 )); then
         return 1
     else
@@ -344,6 +344,7 @@ if [[ "\$PATH_TO_ASAN" ]]; then
   DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN"
 fi
 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
+echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 
 # ASAN and syscall filters aren't compatible with each other.
 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
@@ -354,6 +355,11 @@ JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 mkdir -p "\$JOURNALD_CONF_DIR"
 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 
+# 90s isn't enough for some services to finish when literally everything is run
+# under ASan+UBSan in containers, which, in turn, are run in VMs.
+mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
+printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/timeout.conf
+
 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 exec  $ROOTLIBDIR/systemd "\$@"
 EOF
@@ -453,8 +459,39 @@ EOF
     fi
 }
 
+check_asan_reports() {
+    local ret=0
+    local root="$1"
+
+    if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
+        ls -l "$root"
+        if [[ -e "$root/systemd.asan.log.1" ]]; then
+            cat "$root/systemd.asan.log.1"
+            ret=$(($ret+1))
+        fi
+
+	journald_report=$(find "$root" -name "systemd-journald.asan.log*" -exec cat {} \;)
+	if [[ ! -z "$journald_report" ]]; then
+            printf "%s" "$journald_report"
+            ret=$(($ret+1))
+	fi
+
+	pids=$("$BUILD_DIR/journalctl" -D "$root/var/log/journal" | perl -alne 'print $1 if /\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/')
+	if [[ ! -z "$pids" ]]; then
+            ret=$(($ret+1))
+            for pid in $pids; do
+                "$BUILD_DIR/journalctl" -D "$root/var/log/journal" _PID=$pid --no-pager
+            done
+	fi
+    fi
+
+    return $ret
+}
+
 check_result_nspawn() {
     local ret=1
+    local journald_report=""
+    local pids=""
     [[ -e $TESTDIR/$1/testok ]] && ret=0
     [[ -f $TESTDIR/$1/failed ]] && cp -a $TESTDIR/$1/failed $TESTDIR
     cp -a $TESTDIR/$1/var/log/journal $TESTDIR
@@ -462,6 +499,7 @@ check_result_nspawn() {
     ls -l $TESTDIR/journal/*/*.journal
     test -s $TESTDIR/failed && ret=$(($ret+1))
     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
+    check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
     return $ret
 }
 
@@ -473,6 +511,7 @@ check_result_qemu() {
     [[ -e $TESTDIR/root/testok ]] && ret=0
     [[ -f $TESTDIR/root/failed ]] && cp -a $TESTDIR/root/failed $TESTDIR
     cp -a $TESTDIR/root/var/log/journal $TESTDIR
+    check_asan_reports "$TESTDIR/root" || ret=$(($ret+1))
     umount $TESTDIR/root
     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
     ls -l $TESTDIR/journal/*/*.journal

travis-ci/managers/debian.sh

@@ -39,22 +39,16 @@ for phase in "${PHASES[@]}"; do
             $DOCKER_EXEC apt-get -y update
             $DOCKER_EXEC apt-get -y build-dep systemd
             $DOCKER_EXEC apt-get -y install "${ADDITIONAL_DEPS[@]}"
-            # overlayfs on TravisCI is having trouble delivering inotify events to test-path and test-event.
-            # Let's use tmpfs instead for now.
-            $DOCKER_EXEC mount -t tmpfs tmpfs /tmp
             ;;
-        RUN)
-            info "Run phase"
-            $DOCKER_EXEC meson --werror -Dtests=unsafe -Dslow-tests=true -Dsplit-usr=true build
+        RUN|RUN_CLANG)
+            if [[ "$phase" = "RUN_CLANG" ]]; then
+                ENV_VARS="-e CC=clang -e CXX=clang++"
+            fi
+            docker exec $ENV_VARS -it $CONT_NAME meson --werror -Dtests=unsafe -Dslow-tests=true -Dsplit-usr=true build
             $DOCKER_EXEC ninja -v -C build
-            $DOCKER_EXEC ninja -C build test
+            docker exec -e "TRAVIS=$TRAVIS" -it $CONT_NAME ninja -C build test
             $DOCKER_EXEC tools/check-directives.sh
             ;;
-        RUN_CLANG)
-            docker exec -e CC=clang -e CXX=clang++ -it $CONT_NAME meson --werror -Dtests=unsafe -Dslow-tests=true -Dsplit-usr=true build
-            $DOCKER_EXEC ninja -v -C build
-            $DOCKER_EXEC ninja -C build test
-            ;;
         RUN_ASAN|RUN_CLANG_ASAN)
             if [[ "$phase" = "RUN_CLANG_ASAN" ]]; then
                 ENV_VARS="-e CC=clang -e CXX=clang++"

travis-ci/managers/xenial.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+set -x
+
+add-apt-repository ppa:pitti/systemd-semaphore -y
+apt-get update
+apt-get build-dep systemd -y
+apt-get install -y util-linux libmount-dev libblkid-dev liblzma-dev libqrencode-dev libmicrohttpd-dev iptables-dev liblz4-dev libcurl4-gnutls-dev unifont itstool kbd cryptsetup-bin net-tools isc-dhcp-client iputils-ping strace qemu-system-x86 linux-image-virtual mount libgpg-error-dev libxkbcommon-dev python-lxml python3-lxml python3-pip libcap-dev
+apt-get install -y gettext python3-evdev python3-pyparsing libmount-dev python3-setuptools ninja-build
+pip3 install meson
+
+cd $REPO_ROOT
+
+sed -i 's/2\.30/2.27/' meson.build
+
+meson --werror -Db_sanitize=address,undefined -Dsplit-usr=true build
+ninja -v -C build
+make -C test/TEST-01-BASIC clean setup run TEST_NO_QEMU=yes NSPAWN_ARGUMENTS=--keep-unit RUN_IN_UNPRIVILEGED_CONTAINER=no
+
+# Now that we're more or less sure that ASan isn't going to crash systemd and cause a kernel panic
+# let's also run the test with QEMU to cover udevd, sysctl and everything else that isn't run
+# in containers.
+make -C test/TEST-01-BASIC clean setup run TEST_NO_NSPAWN=yes