From 974fe6131f1fae5c31e18e0979a40d56a85c2c88 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Fri, 9 Feb 2024 18:44:58 +0100 Subject: [PATCH 1/2] test: make the MemoryHigh= limit a bit more generous with sanitizers When we're running with sanitizers, sd-executor might pull in a significant chunk of shared libraries on startup, that can cause a lot of memory pressure and put us in the front when sd-oomd decides to go on a killing spree. This is exacerbated further on Arch Linux when built with gcc, as Arch ships unstripped gcc-libs so sd-executor pulls in over 30M of additional shared libs on startup: ~# lddtree build-san/systemd-executor build-san/systemd-executor (interpreter => /lib64/ld-linux-x86-64.so.2) libasan.so.8 => /usr/lib/libasan.so.8 libstdc++.so.6 => /usr/lib/libstdc++.so.6 libm.so.6 => /usr/lib/libm.so.6 libgcc_s.so.1 => /usr/lib/libgcc_s.so.1 libsystemd-core-255.so => /root/systemd/build-san/src/core/libsystemd-core-255.so libaudit.so.1 => /usr/lib/libaudit.so.1 libcap-ng.so.0 => /usr/lib/libcap-ng.so.0 ... libseccomp.so.2 => /usr/lib/libseccomp.so.2 libubsan.so.1 => /usr/lib/libubsan.so.1 libc.so.6 => /usr/lib/libc.so.6 ~# ls -Llh /usr/lib/libasan.so.8 /usr/lib/libstdc++.so.6 /usr/lib/libubsan.so.1 -rwxr-xr-x 1 root root 9.7M Feb 2 10:36 /usr/lib/libasan.so.8 -rwxr-xr-x 1 root root 21M Feb 2 10:36 /usr/lib/libstdc++.so.6 -rwxr-xr-x 1 root root 3.2M Feb 2 10:36 /usr/lib/libubsan.so.1 Sanitized libsystemd-core.so is also quite big: ~# ls -Llh /root/systemd/build-san/src/core/libsystemd-core-255.so /usr/lib/systemd/libsystemd-core-255.so -rwxr-xr-x 1 root root 26M Feb 8 19:04 /root/systemd/build-san/src/core/libsystemd-core-255.so -rwxr-xr-x 1 root root 5.9M Feb 7 12:03 /usr/lib/systemd/libsystemd-core-255.so --- test/units/testsuite-55.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/units/testsuite-55.sh b/test/units/testsuite-55.sh index afd3053e4da..251a7723038 100755 --- a/test/units/testsuite-55.sh +++ b/test/units/testsuite-55.sh @@ -71,9 +71,22 @@ if systemctl is-active systemd-oomd.service; then systemctl restart systemd-oomd.service fi -# Ensure that we can start services even with a very low hard memory cap without oom-kills, but skip under -# sanitizers as they balloon memory usage. -if ! [[ -v ASAN_OPTIONS || -v UBSAN_OPTIONS ]]; then +if [[ -v ASAN_OPTIONS || -v UBSAN_OPTIONS ]]; then + # If we're running with sanitizers, sd-executor might pull in quite a significant chunk of shared + # libraries, which in turn causes a lot of pressure that can put us in the front when sd-oomd decides to + # go on a killing spree. This fact is exacerbated further on Arch Linux which ships unstripped gcc-libs, + # so sd-executor pulls in over 30M of libs on startup. Let's make the MemoryHigh= limit a bit more + # generous when running with sanitizers to make the test happy. + systemctl edit --runtime --stdin --drop-in=99-MemoryHigh.conf testsuite-55-testchill.service < Date: Fri, 9 Feb 2024 18:53:19 +0100 Subject: [PATCH 2/2] test: clean up the code a bit --- test/units/testsuite-55.sh | 56 ++++++++++---------------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/test/units/testsuite-55.sh b/test/units/testsuite-55.sh index 251a7723038..2b1383ecade 100755 --- a/test/units/testsuite-55.sh +++ b/test/units/testsuite-55.sh @@ -13,15 +13,9 @@ test "$(cat /sys/fs/cgroup/init.scope/memory.high)" != "max" # Loose checks to ensure the environment has the necessary features for systemd-oomd [[ -e /proc/pressure ]] || echo "no PSI" >>/skipped -cgroup_type="$(stat -fc %T /sys/fs/cgroup/)" -if [[ "$cgroup_type" != *"cgroup2"* ]] && [[ "$cgroup_type" != *"0x63677270"* ]]; then - echo "no cgroup2" >>/skipped -fi -if [ ! -f /usr/lib/systemd/systemd-oomd ] && [ ! -f /lib/systemd/systemd-oomd ]; then - echo "no oomd" >>/skipped -fi - -if [[ -e /skipped ]]; then +[[ "$(get_cgroup_hierarchy)" == "unified" ]] || echo "no cgroupsv2" >>/skipped +[[ -x /usr/lib/systemd/systemd-oomd ]] || echo "no oomd" >>/skipped +if [[ -s /skipped ]]; then exit 0 fi @@ -94,27 +88,16 @@ systemctl start testsuite-55-testchill.service systemctl start testsuite-55-testbloat.service # Verify systemd-oomd is monitoring the expected units -# Try to avoid racing the oomctl output check by checking in a loop with a timeout -oomctl_output=$(oomctl) -timeout="$(date -ud "1 minutes" +%s)" -while [[ $(date -u +%s) -le $timeout ]]; do - if grep "/testsuite-55-workload.slice" <<< "$oomctl_output"; then - break - fi - oomctl_output=$(oomctl) - sleep 1 -done - -grep "/testsuite-55-workload.slice" <<< "$oomctl_output" -grep "20.00%" <<< "$oomctl_output" -grep "Default Memory Pressure Duration: 2s" <<< "$oomctl_output" +timeout 1m bash -xec 'until oomctl | grep "/testsuite-55-workload.slice"; do sleep 1; done' +oomctl | grep "/testsuite-55-workload.slice" +oomctl | grep "20.00%" +oomctl | grep "Default Memory Pressure Duration: 2s" systemctl status testsuite-55-testchill.service # systemd-oomd watches for elevated pressure for 2 seconds before acting. # It can take time to build up pressure so either wait 2 minutes or for the service to fail. -timeout="$(date -ud "2 minutes" +%s)" -while [[ $(date -u +%s) -le $timeout ]]; do +for _ in {0..59}; do if ! systemctl status testsuite-55-testbloat.service; then break fi @@ -133,26 +116,16 @@ systemctl start --machine "testuser@.host" --user testsuite-55-testbloat.service # Verify systemd-oomd is monitoring the expected units # Try to avoid racing the oomctl output check by checking in a loop with a timeout -oomctl_output=$(oomctl) -timeout="$(date -ud "1 minutes" +%s)" -while [[ $(date -u +%s) -le $timeout ]]; do - if grep -E "/user.slice.*/testsuite-55-workload.slice" <<< "$oomctl_output"; then - break - fi - oomctl_output=$(oomctl) - sleep 1 -done - -grep -E "/user.slice.*/testsuite-55-workload.slice" <<< "$oomctl_output" -grep "20.00%" <<< "$oomctl_output" -grep "Default Memory Pressure Duration: 2s" <<< "$oomctl_output" +timeout 1m bash -xec 'until oomctl | grep "/testsuite-55-workload.slice"; do sleep 1; done' +oomctl | grep -E "/user.slice.*/testsuite-55-workload.slice" +oomctl | grep "20.00%" +oomctl | grep "Default Memory Pressure Duration: 2s" systemctl --machine "testuser@.host" --user status testsuite-55-testchill.service # systemd-oomd watches for elevated pressure for 2 seconds before acting. # It can take time to build up pressure so either wait 2 minutes or for the service to fail. -timeout="$(date -ud "2 minutes" +%s)" -while [[ $(date -u +%s) -le $timeout ]]; do +for _ in {0..59}; do if ! systemctl --machine "testuser@.host" --user status testsuite-55-testbloat.service; then break fi @@ -179,8 +152,7 @@ EOF systemctl start testsuite-55-testmunch.service systemctl start testsuite-55-testbloat.service - timeout="$(date -ud "2 minutes" +%s)" - while [[ "$(date -u +%s)" -le "$timeout" ]]; do + for _ in {0..59}; do if ! systemctl status testsuite-55-testmunch.service; then break fi