2020-03-04 12:35:06 +03:00
#!/usr/bin/env bash
2021-10-17 19:13:06 +03:00
# SPDX-License-Identifier: LGPL-2.1-or-later
2021-04-09 20:39:41 +03:00
set -eux
2019-06-26 00:01:40 +03:00
set -o pipefail
at_exit( ) {
2021-04-09 20:56:12 +03:00
# shellcheck disable=SC2181
if [ [ $? -ne 0 ] ] ; then
2019-06-26 00:01:40 +03:00
# We're exiting with a non-zero EC, let's dump test artifacts
# for easier debugging
2021-04-09 20:56:12 +03:00
[ [ -v straceLog && -f " $straceLog " ] ] && cat " $straceLog "
[ [ -v journalLog && -f " $journalLog " ] ] && cat " $journalLog "
2019-06-26 00:01:40 +03:00
fi
}
trap at_exit EXIT
systemd-analyze log-level debug
systemd-analyze log-target journal
# Log files
straceLog = 'strace.log'
journalLog = 'journal.log'
# Systemd config files
testUnit = 'numa-test.service'
2020-03-21 13:17:30 +03:00
testUnitFile = " /run/systemd/system/ $testUnit "
2019-06-26 00:01:40 +03:00
testUnitNUMAConf = " $testUnitFile .d/numa.conf "
# Sleep constants (we should probably figure out something better but nothing comes to mind)
sleepAfterStart = 1
2019-07-01 10:27:59 +03:00
# Journal cursor for easier navigation
journalCursorFile = "jounalCursorFile"
2019-06-26 00:01:40 +03:00
startStrace( ) {
2021-04-09 20:56:12 +03:00
coproc strace -qq -p 1 -o " $straceLog " -e set_mempolicy -s 1024 ${ 1 : + " $1 " }
2021-07-09 16:22:41 +03:00
# Wait for strace to properly "initialize", i.e. until PID 1 has the TracerPid
# field set to the current strace's PID
while ! awk -v spid = " $COPROC_PID " '/^TracerPid:/ {exit !($2 == spid);}' /proc/1/status; do sleep 0.1; done
2019-06-26 00:01:40 +03:00
}
stopStrace( ) {
2021-04-09 20:56:12 +03:00
[ [ -v COPROC_PID ] ] || return
local PID = $COPROC_PID
kill -s TERM " $PID "
2019-07-01 20:53:45 +03:00
# Make sure the strace process is indeed dead
2021-04-09 20:56:12 +03:00
while kill -0 " $PID " 2>/dev/null; do sleep 0.1; done
2019-06-26 00:01:40 +03:00
}
startJournalctl( ) {
2021-04-09 20:56:12 +03:00
: >" $journalCursorFile "
2019-07-01 10:27:59 +03:00
# Save journal's cursor for later navigation
journalctl --no-pager --cursor-file= " $journalCursorFile " -n0 -ocat
2019-06-26 00:01:40 +03:00
}
stopJournalctl( ) {
2019-08-05 15:38:45 +03:00
local unit = " ${ 1 :- init .scope } "
2019-07-01 10:27:59 +03:00
# Using journalctl --sync should be better than using SIGRTMIN+1, as
2019-06-26 00:01:40 +03:00
# the --sync wait until the synchronization is complete
echo "Force journald to write all queued messages"
journalctl --sync
2021-04-09 20:49:32 +03:00
journalctl -u " $unit " --cursor-file= " $journalCursorFile " >" $journalLog "
2019-06-26 00:01:40 +03:00
}
checkNUMA( ) {
# NUMA enabled system should have at least NUMA node0
test -e /sys/devices/system/node/node0
}
writePID1NUMAPolicy( ) {
2021-04-09 20:56:12 +03:00
cat >" $confDir /numa.conf " <<EOF
[ Manager]
NUMAPolicy = ${ 1 : ?missing argument : NUMAPolicy }
NUMAMask = ${ 2 :- "" }
EOF
2019-06-26 00:01:40 +03:00
}
writeTestUnit( ) {
2021-04-09 20:56:12 +03:00
mkdir -p " $testUnitFile .d/ "
printf "[Service]\nExecStart=/bin/sleep 3600\n" >" $testUnitFile "
2019-06-26 00:01:40 +03:00
}
writeTestUnitNUMAPolicy( ) {
2021-04-09 20:56:12 +03:00
cat >" $testUnitNUMAConf " <<EOF
[ Service]
NUMAPolicy = ${ 1 : ?missing argument : NUMAPolicy }
NUMAMask = ${ 2 :- "" }
EOF
2019-06-26 00:01:40 +03:00
systemctl daemon-reload
}
pid1ReloadWithStrace( ) {
startStrace
systemctl daemon-reload
2019-07-01 20:53:45 +03:00
sleep $sleepAfterStart
2019-06-26 00:01:40 +03:00
stopStrace
}
pid1ReloadWithJournal( ) {
startJournalctl
systemctl daemon-reload
stopJournalctl
}
pid1StartUnitWithStrace( ) {
startStrace '-f'
2021-04-09 20:49:32 +03:00
systemctl start " ${ 1 : ?missing unit name } "
2019-06-26 00:01:40 +03:00
sleep $sleepAfterStart
stopStrace
}
pid1StartUnitWithJournal( ) {
startJournalctl
2021-04-09 20:49:32 +03:00
systemctl start " ${ 1 : ?missing unit name } "
2019-06-26 00:01:40 +03:00
sleep $sleepAfterStart
stopJournalctl
}
pid1StopUnit( ) {
2021-04-09 20:49:32 +03:00
systemctl stop " ${ 1 : ?missing unit name } "
2019-06-26 00:01:40 +03:00
}
systemctlCheckNUMAProperties( ) {
2021-04-09 20:49:32 +03:00
local UNIT_NAME = " ${ 1 : ?missing unit name } "
local NUMA_POLICY = " ${ 2 : ?missing NUMAPolicy } "
local NUMA_MASK = " ${ 3 :- "" } "
local LOGFILE
2019-06-26 00:01:40 +03:00
2021-04-09 20:49:32 +03:00
LOGFILE = " $( mktemp) "
2019-06-26 00:01:40 +03:00
2021-04-09 20:49:32 +03:00
systemctl show -p NUMAPolicy " $UNIT_NAME " >" $LOGFILE "
grep " NUMAPolicy= $NUMA_POLICY " " $LOGFILE "
: >" $LOGFILE "
if [ -n " $NUMA_MASK " ] ; then
systemctl show -p NUMAMask " $UNIT_NAME " >" $LOGFILE "
grep " NUMAMask= $NUMA_MASK " " $LOGFILE "
2019-06-26 00:01:40 +03:00
fi
}
writeTestUnit
# Create systemd config drop-in directory
2020-03-21 13:17:30 +03:00
confDir = "/run/systemd/system.conf.d/"
2019-06-26 00:01:40 +03:00
mkdir -p " $confDir "
2019-08-05 15:38:45 +03:00
if ! checkNUMA; then
echo >& 2 "NUMA is not supported on this machine, switching to a simple sanity check"
echo "PID1 NUMAPolicy=default && NUMAMask=0 check without NUMA support"
writePID1NUMAPolicy "default" "0"
startJournalctl
systemctl daemon-reload
stopJournalctl
grep "NUMA support not available, ignoring" " $journalLog "
echo "systemd-run NUMAPolicy=default && NUMAMask=0 check without NUMA support"
runUnit = 'numa-systemd-run-test.service'
startJournalctl
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = default -p NUMAMask = 0 --unit " $runUnit " sleep 1000
2019-08-05 15:38:45 +03:00
sleep $sleepAfterStart
2021-04-09 20:49:32 +03:00
pid1StopUnit " $runUnit "
stopJournalctl " $runUnit "
2019-08-05 15:38:45 +03:00
grep "NUMA support not available, ignoring" " $journalLog "
else
echo "PID1 NUMAPolicy support - Default policy w/o mask"
writePID1NUMAPolicy "default"
pid1ReloadWithStrace
# Kernel requires that nodemask argument is set to NULL when setting default policy
2021-04-09 20:49:32 +03:00
grep "set_mempolicy(MPOL_DEFAULT, NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Default policy w/ mask"
writePID1NUMAPolicy "default" "0"
pid1ReloadWithStrace
2021-04-09 20:49:32 +03:00
grep "set_mempolicy(MPOL_DEFAULT, NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Bind policy w/o mask"
writePID1NUMAPolicy "bind"
pid1ReloadWithJournal
2021-04-09 20:49:32 +03:00
grep "Failed to set NUMA memory policy: Invalid argument" " $journalLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Bind policy w/ mask"
writePID1NUMAPolicy "bind" "0"
pid1ReloadWithStrace
2021-04-09 20:49:32 +03:00
grep -P "set_mempolicy\(MPOL_BIND, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Interleave policy w/o mask"
writePID1NUMAPolicy "interleave"
pid1ReloadWithJournal
2021-04-09 20:49:32 +03:00
grep "Failed to set NUMA memory policy: Invalid argument" " $journalLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Interleave policy w/ mask"
writePID1NUMAPolicy "interleave" "0"
pid1ReloadWithStrace
2021-04-09 20:49:32 +03:00
grep -P "set_mempolicy\(MPOL_INTERLEAVE, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Preferred policy w/o mask"
writePID1NUMAPolicy "preferred"
pid1ReloadWithJournal
# Preferred policy with empty node mask is actually allowed and should reset allocation policy to default
2021-04-09 20:49:32 +03:00
grep "Failed to set NUMA memory policy: Invalid argument" " $journalLog " && { echo >& 2 "unexpected pass" ; exit 1; }
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Preferred policy w/ mask"
writePID1NUMAPolicy "preferred" "0"
pid1ReloadWithStrace
2021-04-09 20:49:32 +03:00
grep -P "set_mempolicy\(MPOL_PREFERRED, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Local policy w/o mask"
writePID1NUMAPolicy "local"
pid1ReloadWithStrace
# Kernel requires that nodemask argument is set to NULL when setting default policy
# The unpatched versions of strace don't recognize the MPOL_LOCAL constant and
# return a numerical constant instead (with a comment):
# set_mempolicy(0x4 /* MPOL_??? */, NULL, 0) = 0
# Let's cover this scenario as well
2021-04-09 20:49:32 +03:00
grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "PID1 NUMAPolicy support - Local policy w/ mask"
writePID1NUMAPolicy "local" "0"
pid1ReloadWithStrace
2021-04-09 20:49:32 +03:00
grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Default policy w/o mask"
writeTestUnitNUMAPolicy "default"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "default"
pid1StopUnit " $testUnit "
grep "set_mempolicy(MPOL_DEFAULT, NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Default policy w/ mask"
writeTestUnitNUMAPolicy "default" "0"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "default" "0"
2019-08-05 15:38:45 +03:00
pid1StopUnit $testUnit
2020-12-16 20:16:16 +03:00
# Mask must be ignored
2021-04-09 20:49:32 +03:00
grep "set_mempolicy(MPOL_DEFAULT, NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Bind policy w/o mask"
writeTestUnitNUMAPolicy "bind"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithJournal " $testUnit "
pid1StopUnit " $testUnit "
grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" " $journalLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Bind policy w/ mask"
writeTestUnitNUMAPolicy "bind" "0"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "bind" "0"
pid1StopUnit " $testUnit "
grep -P "set_mempolicy\(MPOL_BIND, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Interleave policy w/o mask"
writeTestUnitNUMAPolicy "interleave"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
pid1StopUnit " $testUnit "
grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" " $journalLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Interleave policy w/ mask"
writeTestUnitNUMAPolicy "interleave" "0"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "interleave" "0"
pid1StopUnit " $testUnit "
grep -P "set_mempolicy\(MPOL_INTERLEAVE, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Preferred policy w/o mask"
writeTestUnitNUMAPolicy "preferred"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithJournal " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "preferred"
pid1StopUnit " $testUnit "
grep "numa-test.service: Main process exited, code=exited, status=242/NUMA" " $journalLog " && { echo >& 2 "unexpected pass" ; exit 1; }
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Preferred policy w/ mask"
writeTestUnitNUMAPolicy "preferred" "0"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "preferred" "0"
pid1StopUnit " $testUnit "
grep -P "set_mempolicy\(MPOL_PREFERRED, \[0x0*1\]" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Local policy w/o mask"
writeTestUnitNUMAPolicy "local"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "local"
pid1StopUnit " $testUnit "
grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
echo "Unit file NUMAPolicy support - Local policy w/ mask"
writeTestUnitNUMAPolicy "local" "0"
2021-04-09 20:49:32 +03:00
pid1StartUnitWithStrace " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "local" "0"
pid1StopUnit " $testUnit "
2020-12-16 20:16:16 +03:00
# Mask must be ignored
2021-04-09 20:49:32 +03:00
grep -E "set_mempolicy\((MPOL_LOCAL|0x4 [^,]*), NULL" " $straceLog "
2019-08-05 15:38:45 +03:00
2020-02-17 15:50:31 +03:00
echo "Unit file CPUAffinity=NUMA support"
writeTestUnitNUMAPolicy "bind" "0"
2021-04-09 20:49:32 +03:00
echo "CPUAffinity=numa" >>" $testUnitNUMAConf "
2020-02-17 15:50:31 +03:00
systemctl daemon-reload
2021-04-09 20:49:32 +03:00
systemctl start " $testUnit "
systemctlCheckNUMAProperties " $testUnit " "bind" "0"
cpulist = " $( cat /sys/devices/system/node/node0/cpulist) "
affinity_systemd = " $( systemctl show --value -p CPUAffinity " $testUnit " ) "
[ " $cpulist " = " $affinity_systemd " ]
pid1StopUnit " $testUnit "
2020-02-17 15:50:31 +03:00
2019-08-05 15:38:45 +03:00
echo "systemd-run NUMAPolicy support"
runUnit = 'numa-systemd-run-test.service'
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = default --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "default"
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = default -p NUMAMask = 0 --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "default" ""
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = bind -p NUMAMask = 0 --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "bind" "0"
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = interleave -p NUMAMask = 0 --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "interleave" "0"
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = preferred -p NUMAMask = 0 --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "preferred" "0"
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = local --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "local"
pid1StopUnit " $runUnit "
2020-02-17 15:50:31 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = local -p NUMAMask = 0 --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "local" ""
pid1StopUnit " $runUnit "
2020-02-17 15:50:31 +03:00
2021-04-09 20:49:32 +03:00
systemd-run -p NUMAPolicy = local -p NUMAMask = 0 -p CPUAffinity = numa --unit " $runUnit " sleep 1000
systemctlCheckNUMAProperties " $runUnit " "local" ""
systemctl cat " $runUnit " | grep -q 'CPUAffinity=numa'
pid1StopUnit " $runUnit "
2019-08-05 15:38:45 +03:00
fi
2019-06-26 00:01:40 +03:00
# Cleanup
2021-04-09 20:49:32 +03:00
rm -rf " $confDir "
2019-06-26 00:01:40 +03:00
systemctl daemon-reload
systemd-analyze log-level info
2021-04-08 01:09:55 +03:00
echo OK >/testok
2019-06-26 00:01:40 +03:00
exit 0