shaba/systemd-stable
1
1
Fork 0
mirror of https://github.com/systemd/systemd-stable.git synced 2024-12-27 03:21:32 +03:00
Code
80b44b38b5
systemd-stable/test/test-functions
Frantisek Sumsal 80b44b38b5 test: correctly clean up test artifacts 
Remove the artifact files indicating test result (testok, failed, and
skipped) just before running the test so we always get the latest and
most relevant result instead of incorrectly consuming previous results.

Discovered in https://github.com/systemd/systemd/pull/15378#issuecomment-616801873
2020-04-22 08:57:38 +02:00

2116 lines
64 KiB
Bash
Raw Blame History

#!/usr/bin/env bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin
export PATH
LOOKS_LIKE_DEBIAN=$(source /etc/os-release && [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && echo yes || :)
LOOKS_LIKE_ARCH=$(source /etc/os-release && [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && echo yes || :)
LOOKS_LIKE_SUSE=$(source /etc/os-release && [[ " $ID_LIKE " = *" suse "* ]] && echo yes || :)
KERNEL_VER=${KERNEL_VER-$(uname -r)}
KERNEL_MODS="/lib/modules/$KERNEL_VER/"
QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
TIMED_OUT= # will be 1 after run_* if *_TIMEOUT is set and test timed out
[[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
QEMU_MEM="${QEMU_MEM:-512M}"
IMAGE_NAME=${IMAGE_NAME:-default}
TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
LOOPDEV=
# Decide if we can (and want to) run QEMU with KVM acceleration.
# Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
# check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
# running under KVM. If these conditions are met, enable KVM (and possibly
# nested KVM), otherwise disable it.
if [[ -n "$TEST_NESTED_KVM" || ( -z "$TEST_NO_KVM" && $(systemd-detect-virt -v) != kvm ) ]]; then
QEMU_KVM=yes
else
QEMU_KVM=no
fi
if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
ROOTLIBDIR=/usr/lib/systemd
fi
PATH_TO_INIT=$ROOTLIBDIR/systemd
[ "$SYSTEMD_JOURNALD" ] || SYSTEMD_JOURNALD=$(which -a $BUILD_DIR/systemd-journald $ROOTLIBDIR/systemd-journald 2>/dev/null | grep '^/' -m1)
[ "$SYSTEMD" ] || SYSTEMD=$(which -a $BUILD_DIR/systemd $ROOTLIBDIR/systemd 2>/dev/null | grep '^/' -m1)
[ "$SYSTEMD_NSPAWN" ] || SYSTEMD_NSPAWN=$(which -a $BUILD_DIR/systemd-nspawn systemd-nspawn 2>/dev/null | grep '^/' -m1)
[ "$JOURNALCTL" ] || JOURNALCTL=$(which -a $BUILD_DIR/journalctl journalctl 2>/dev/null | grep '^/' -m1)
BASICTOOLS=(
awk
basename
bash
busybox
capsh
cat
chmod
chown
cmp
cryptsetup
cut
date
dd
diff
dirname
dmsetup
echo
env
false
getent
getfacl
grep
gunzip
gzip
head
ionice
ip
ln
loadkeys
login
lz4cat
mkfifo
mktemp
modprobe
mount
mv
nc
nproc
readlink
rev
rm
rmdir
sed
seq
setfont
setsid
sfdisk
sh
sleep
socat
stat
su
sulogin
sysctl
tail
tar
tee
test
touch
tr
true
truncate
umount
uname
unshare
xargs
xzcat
)
DEBUGTOOLS=(
cp
df
dhclient
dmesg
du
find
free
grep
hostname
id
less
ln
ls
mkdir
ping
ps
route
sort
strace
stty
tty
vi
)
STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
STATEFILE="$STATEDIR/.testdir"
IMAGESTATEDIR="$STATEDIR/.."
TESTLOG="$STATEDIR/test.log"
is_built_with_asan() {
if ! type -P objdump >/dev/null; then
ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
return 1
fi
# Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
local _asan_calls=$(objdump -dC $SYSTEMD_JOURNALD | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
if (( $_asan_calls < 1000 )); then
return 1
else
return 0
fi
}
IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
STRIP_BINARIES=no
SKIP_INITRD="${SKIP_INITRD:-yes}"
PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
QEMU_MEM="2048M"
QEMU_SMP=4
# We need to correctly distinguish between gcc's and clang's ASan DSOs.
if ldd $SYSTEMD | grep -q libasan.so; then
ASAN_COMPILER=gcc
elif ldd $SYSTEMD | grep -q libclang_rt.asan; then
ASAN_COMPILER=clang
# As clang's ASan DSO is usually in a non-standard path, let's check if
# the environment is set accordingly. If not, warn the user and exit.
# We're not setting the LD_LIBRARY_PATH automagically here, because
# user should encounter (and fix) the same issue when running the unit
# tests (meson test)
if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
_asan_rt_name="$(ldd $SYSTEMD | awk '/libclang_rt.asan/ {print $1; exit}')"
_asan_rt_path="$(find /usr/lib* /usr/local/lib* -type f -name "$_asan_rt_name" 2>/dev/null | sed 1q)"
echo >&2 "clang's ASan DSO ($_asan_rt_name) is not present in the runtime library path"
echo >&2 "Consider setting LD_LIBRARY_PATH=${_asan_rt_path%/*}"
exit 1
fi
else
echo >&2 "systemd is not linked against the ASan DSO"
echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
exit 1
fi
fi
function find_qemu_bin() {
# SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
if [[ $QEMU_KVM == "yes" ]]; then
[ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)
fi
[ "$ARCH" ] || ARCH=$(uname -m)
case $ARCH in
x86_64)
# QEMU's own build system calls it qemu-system-x86_64
[ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
;;
i*86)
# new i386 version of QEMU
[ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)
# i386 version of QEMU
[ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
;;
ppc64*)
[ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-ppc64 2>/dev/null | grep '^/' -m1)
;;
esac
if [ ! -e "$QEMU_BIN" ]; then
echo "Could not find a suitable QEMU binary" >&2
return 1
fi
}
# Return 0 if QEMU did run (then you must check the result state/logs for actual
# success), or 1 if QEMU is not available.
run_qemu() {
if [ -f /etc/machine-id ]; then
read MACHINE_ID < /etc/machine-id
[ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
&& INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
[ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
&& KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
fi
CONSOLE=ttyS0
rm -f "$initdir"/{testok,failed,skipped}
# make sure the initdir is not mounted to avoid concurrent access
cleanup_initdir
umount_loopback
if [[ ! "$KERNEL_BIN" ]]; then
if [[ "$LOOKS_LIKE_ARCH" ]]; then
KERNEL_BIN=/boot/vmlinuz-linux
else
[ "$ARCH" ] || ARCH=$(uname -m)
case $ARCH in
ppc64*)
KERNEL_BIN=/boot/vmlinux-$KERNEL_VER
CONSOLE=hvc0
;;
*)
KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
;;
esac
fi
fi
default_fedora_initrd=/boot/initramfs-${KERNEL_VER}.img
default_debian_initrd=/boot/initrd.img-${KERNEL_VER}
default_arch_initrd=/boot/initramfs-linux-fallback.img
default_suse_initrd=/boot/initrd-${KERNEL_VER}
if [[ ! "$INITRD" ]]; then
if [[ -e "$default_fedora_initrd" ]]; then
INITRD="$default_fedora_initrd"
elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
INITRD="$default_debian_initrd"
elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
INITRD="$default_arch_initrd"
elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
INITRD="$default_suse_initrd"
fi
fi
# If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
# i.e. use the number of online CPUs on the host machine. If the nproc utility
# is not installed or there's some other error when calling it, fall back
# to the original value (QEMU_SMP=1).
if ! [ "$QEMU_SMP" ]; then
if ! QEMU_SMP=$(nproc); then
dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
QEMU_SMP=1
fi
fi
find_qemu_bin || return 1
# Umount initdir to avoid concurrent access to the filesystem
_umount_dir $initdir
local _cgroup_args
if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
_cgroup_args="systemd.unified_cgroup_hierarchy=yes"
elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
_cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=yes"
elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
_cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=no"
elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
exit 1
fi
if [[ "$LOOKS_LIKE_SUSE" ]]; then
PARAMS+="rd.hostonly=0"
fi
local _end
if [[ ! "$INTERACTIVE_DEBUG" ]]; then
_end="systemd.wants=end.service"
else
_end=""
fi
KERNEL_APPEND="$PARAMS \
root=/dev/sda1 \
rw \
raid=noautodetect \
rd.luks=0 \
loglevel=2 \
init=$PATH_TO_INIT \
console=$CONSOLE \
selinux=0 \
$_cgroup_args \
SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units: \
systemd.unit=testsuite.target \
systemd.wants=testsuite-$1.service ${_end} \
$KERNEL_APPEND \
"
QEMU_OPTIONS="-smp $QEMU_SMP \
-net none \
-m $QEMU_MEM \
-nographic \
-kernel $KERNEL_BIN \
-drive format=raw,cache=unsafe,file=${IMAGESTATEDIR}/${IMAGE_NAME}.img \
$QEMU_OPTIONS \
"
if [[ "$INITRD" && "$SKIP_INITRD" != "yes" ]]; then
QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
fi
# Let's use KVM if possible
if [[ -c /dev/kvm && $QEMU_KVM == "yes" ]]; then
QEMU_OPTIONS="$QEMU_OPTIONS -machine accel=kvm -enable-kvm -cpu host"
fi
if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
QEMU_BIN="timeout --foreground $QEMU_TIMEOUT $QEMU_BIN"
fi
(set -x; $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND")
rc=$?
if [ "$rc" = 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
derror "test timed out after $QEMU_TIMEOUT s"
TIMED_OUT=1
else
[ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
fi
return 0
}
# Return 0 if nspawn did run (then you must check the result state/logs for actual
# success), or 1 if nspawn is not available.
run_nspawn() {
[[ -d /run/systemd/system ]] || return 1
rm -f "$initdir"/{testok,failed,skipped}
local _nspawn_cmd=(
--register=no
--kill-signal=SIGKILL
--directory=$1
--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:
$PATH_TO_INIT
$KERNEL_APPEND
systemd.unit=testsuite.target
systemd.wants=testsuite-$2.service
)
if [[ ! "$INTERACTIVE_DEBUG" ]]; then
_nspawn_cmd+=( systemd.wants=end.service )
fi
local _nspawn_pre
if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
_nspawn_pre=(timeout --foreground $NSPAWN_TIMEOUT)
else
_nspawn_pre=()
fi
if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
exit
elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
_nspawn_pre=("${nspawn_pre[@]}" env SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY)
elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
_nspawn_pre=("${nspawn_pre[@]}" env --unset=UNIFIED_CGROUP_HIERARCHY --unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY)
else
dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
exit 1
fi
(set -x; "${_nspawn_pre[@]}" "$SYSTEMD_NSPAWN" $NSPAWN_ARGUMENTS "${_nspawn_cmd[@]}")
rc=$?
if [ "$rc" = 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
derror "test timed out after $NSPAWN_TIMEOUT s"
TIMED_OUT=1
else
[ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
fi
return 0
}
setup_basic_environment() {
# create the basic filesystem layout
setup_basic_dirs
install_systemd
install_missing_libraries
install_config_files
install_zoneinfo
create_rc_local
install_basic_tools
install_libnss
install_pam
install_dbus
install_fonts
install_keymaps
install_terminfo
install_execs
install_fsck
install_plymouth
install_debug_tools
install_ld_so_conf
install_testuser
has_user_dbus_socket && install_user_dbus
setup_selinux
strip_binaries
install_depmod_files
generate_module_dependencies
if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
create_asan_wrapper
fi
}
setup_selinux() {
# don't forget KERNEL_APPEND='... selinux=1 ...'
if [[ "$SETUP_SELINUX" != "yes" ]]; then
ddebug "Don't setup SELinux"
return 0
fi
ddebug "Setup SELinux"
local _conf_dir=/etc/selinux
local _fixfiles_tools="bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles"
rm -rf $initdir/$_conf_dir
if ! cp -ar $_conf_dir $initdir/$_conf_dir; then
dfatal "Failed to copy $_conf_dir"
exit 1
fi
touch $initdir/.autorelabel
mkdir -p $initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants
ln -sf ../autorelabel.service $initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/
dracut_install $_fixfiles_tools
dracut_install fixfiles
dracut_install sestatus
}
install_valgrind() {
if ! type -p valgrind; then
dfatal "Failed to install valgrind"
exit 1
fi
local _valgrind_bins=$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
dracut_install $_valgrind_bins
local _valgrind_libs=$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
dracut_install $_valgrind_libs
local _valgrind_dbg_and_supp=$(
strace -e open valgrind /bin/true 2>&1 >/dev/null |
perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
)
dracut_install $_valgrind_dbg_and_supp
}
create_valgrind_wrapper() {
local _valgrind_wrapper=$initdir/$ROOTLIBDIR/systemd-under-valgrind
ddebug "Create $_valgrind_wrapper"
cat >$_valgrind_wrapper <<EOF
#!/usr/bin/env bash
mount -t proc proc /proc
exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
EOF
chmod 0755 $_valgrind_wrapper
}
create_asan_wrapper() {
local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan
local _asan_rt_pattern
ddebug "Create $_asan_wrapper"
case "$ASAN_COMPILER" in
gcc)
_asan_rt_pattern="*libasan*"
;;
clang)
_asan_rt_pattern="libclang_rt.asan-*"
# Install llvm-symbolizer to generate useful reports
# See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
dracut_install "llvm-symbolizer"
;;
*)
dfail "Unsupported compiler: $ASAN_COMPILER"
exit 1
esac
cat >$_asan_wrapper <<EOF
#!/usr/bin/env bash
set -x
DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"
# As right now bash is the PID 1, we can't expect PATH to have a sane value.
# Let's make one to prevent unexpected "<bin> not found" issues in the future
export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -o remount,rw /
PATH_TO_ASAN=\$(find / -name '$_asan_rt_pattern' | sed 1q)
if [[ "\$PATH_TO_ASAN" ]]; then
# A lot of services (most notably dbus) won't start without preloading libasan
# See https://github.com/systemd/systemd/issues/5004
DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN"
# Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
# unnecessary for gcc & libasan, however, for clang this is crucial, as its
# runtime ASan DSO is in a non-standard (library) path.
echo \${PATH_TO_ASAN%/*} > /etc/ld.so.conf.d/asan-path-override.conf
ldconfig
fi
echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
# ASAN and syscall filters aren't compatible with each other.
find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
# The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
# But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
mkdir -p "\$JOURNALD_CONF_DIR"
printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
# Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
# Let's try to catch them by redirecting stderr (and stdout just in case) to a file
# See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
# 90s isn't enough for some services to finish when literally everything is run
# under ASan+UBSan in containers, which, in turn, are run in VMs.
# Let's limit which environments such services should be executed in.
mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
# Let's override another hard-coded timeout that kicks in too early
mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
# The 'mount' utility doesn't behave well under libasan, causing unexpected
# fails during boot and subsequent test results check:
# bash-5.0# mount -o remount,rw -v /
# mount: /dev/sda1 mounted on /.
# bash-5.0# echo \$?
# 1
# Let's workaround this by clearing the previously set LD_PRELOAD env variable,
# so the libasan library is not loaded for this particular service
unset_ld_preload() {
local _dropin_dir="/etc/systemd/system/\$1.service.d"
mkdir -p "\$_dropin_dir"
printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$_dropin_dir/unset_ld_preload.conf"
}
unset_ld_preload systemd-remount-fs
unset_ld_preload testsuite-
export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
exec $ROOTLIBDIR/systemd "\$@"
EOF
chmod 0755 $_asan_wrapper
}
create_strace_wrapper() {
local _strace_wrapper=$initdir/$ROOTLIBDIR/systemd-under-strace
ddebug "Create $_strace_wrapper"
cat >$_strace_wrapper <<EOF
#!/usr/bin/env bash
exec strace -D -o /strace.out $ROOTLIBDIR/systemd "\$@"
EOF
chmod 0755 $_strace_wrapper
}
install_fsck() {
dracut_install /sbin/fsck*
dracut_install -o /bin/fsck*
# fskc.reiserfs calls reiserfsck. so, install it
dracut_install -o reiserfsck
}
install_dmevent() {
instmods dm_crypt =crypto
inst_binary dmeventd
if [[ "$LOOKS_LIKE_DEBIAN" ]]; then
# dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
# and since buster/bionic 95-dm-notify.rules
# see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
else
inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
fi
}
install_systemd() {
# install compiled files
local _ninja_bin=$(type -P ninja || type -P ninja-build)
if [[ -z "$_ninja_bin" ]]; then
dfatal "ninja was not found"
exit 1
fi
(set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install)
# remove unneeded documentation
rm -fr $initdir/usr/share/{man,doc}
# we strip binaries since debug symbols increase binaries size a lot
# and it could fill the available space
strip_binaries
[[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
# enable debug logging in PID1
echo LogLevel=debug >> $initdir/etc/systemd/system.conf
# store coredumps in journal
echo Storage=journal >> $initdir/etc/systemd/coredump.conf
}
get_ldpath() {
local _bin="$1"
local rpath=$(objdump -p "$_bin" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)
if [ -z "$rpath" ] ; then
echo $BUILD_DIR
else
echo $rpath
fi
}
install_missing_libraries() {
# install possible missing libraries
for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath $i)" inst_libs $i
done
}
cleanup_loopdev() {
if [ -n "${LOOPDEV}" ]; then
ddebug "losetup -d $LOOPDEV"
losetup -d "${LOOPDEV}"
fi
}
trap cleanup_loopdev EXIT INT QUIT PIPE
create_empty_image() {
if [ -z "$IMAGE_NAME" ]; then
echo "create_empty_image: \$IMAGE_NAME not set"
exit 1
fi
local _size=500
if [[ "$STRIP_BINARIES" = "no" ]]; then
_size=$((4*_size))
fi
image="${TESTDIR}/${IMAGE_NAME}.img"
public="$IMAGESTATEDIR/${IMAGE_NAME}.img"
echo "Setting up $public (${_size} MB)"
rm -f "$image" "$public"
# Create the blank file to use as a root filesystem
truncate -s "${_size}M" "$image"
ln -vs "$(realpath $image)" "$public"
LOOPDEV=$(losetup --show -P -f "$public")
[ -b "$LOOPDEV" ] || return 1
sfdisk "$LOOPDEV" <<EOF
,$((_size-50))M
,
EOF
udevadm settle
local _label="-L systemd.${name}"
# mkfs.reiserfs doesn't know -L. so, use --label instead
[[ "$FSTYPE" == "reiserfs" ]] && _label="--label systemd.${name}"
mkfs -t "${FSTYPE}" ${_label} "${LOOPDEV}p1" -q; ret=$?
if [ $ret -ne 0 ] ; then
dfatal "Failed to mkfs -t ${FSTYPE}"
exit 1
fi
}
mount_initdir() {
if [ -z "${LOOPDEV}" ]; then
image="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
LOOPDEV=$(losetup --show -P -f "$image")
[ -b "$LOOPDEV" ] || return 1
udevadm settle
fi
if ! mountpoint -q $initdir; then
mkdir -p $initdir
mount ${LOOPDEV}p1 $initdir
TEST_SETUP_CLEANUP_ROOTDIR=1
fi
}
cleanup_initdir() {
# only umount if create_empty_image_rootdir() was called to mount it
[[ -z $TEST_SETUP_CLEANUP_ROOTDIR ]] || _umount_dir $initdir
}
umount_loopback() {
# unmount the loopback device from all places. Otherwise we risk file
# system corruption.
image="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
for device in $(losetup -l | awk '$6=="'"$image"'" {print $1}'); do
ddebug "Unmounting all uses of $device"
mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
done
}
create_empty_image_rootdir() {
create_empty_image
mount_initdir
}
check_asan_reports() {
local ret=0
local root="$1"
if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
ls -l "$root"
if [[ -e "$root/systemd.asan.log.1" ]]; then
cat "$root/systemd.asan.log.1"
ret=$(($ret+1))
fi
journald_report=$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)
if [[ ! -z "$journald_report" ]]; then
printf "%s\n" "$journald_report"
cat "$root/systemd-journald.out" || :
ret=$(($ret+1))
fi
pids=$(
"$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
BEGIN {
%services_to_ignore = (
"dbus-daemon" => undef,
);
}
print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
)
if [[ ! -z "$pids" ]]; then
ret=$(($ret+1))
for pid in $pids; do
"$JOURNALCTL" -D "$root/var/log/journal" _PID=$pid --no-pager
done
fi
fi
return $ret
}
save_journal() {
if [ -n "${ARTIFACT_DIRECTORY}" ]; then
dest="${ARTIFACT_DIRECTORY}/${testname}.journal"
else
dest="$TESTDIR/system.journal"
fi
for j in $1/*; do
/usr/lib/systemd/systemd-journal-remote \
-o $dest \
--getter="journalctl -o export -D $j"
if [ -n "${TEST_SHOW_JOURNAL}" ]; then
echo "---- $j ----"
journalctl --no-pager -o short-monotonic --no-hostname --priority=${TEST_SHOW_JOURNAL} -D $j
fi
rm -r $j
done
# we want to print this sometime later, so save this in a variable
JOURNAL_LIST="$(ls -l $dest*)"
}
check_result_nspawn() {
local ret=1
local journald_report=""
local pids=""
[[ -e $1/testok ]] && ret=0
[[ -f $1/failed ]] && cp -a $1/failed $TESTDIR
save_journal $1/var/log/journal
[[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
echo $JOURNAL_LIST
test -s $TESTDIR/failed && ret=$(($ret+1))
[ -n "$TIMED_OUT" ] && ret=$(($ret+1))
check_asan_reports "$1" || ret=$(($ret+1))
_umount_dir $initdir
return $ret
}
# can be overridden in specific test
check_result_qemu() {
local ret=1
mount_initdir
[[ -e $initdir/testok ]] && ret=0
[[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
save_journal $initdir/var/log/journal
check_asan_reports "$initdir" || ret=$(($ret+1))
_umount_dir $initdir
[[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
echo $JOURNAL_LIST
test -s $TESTDIR/failed && ret=$(($ret+1))
[ -n "$TIMED_OUT" ] && ret=$(($ret+1))
return $ret
}
strip_binaries() {
if [[ "$STRIP_BINARIES" = "no" ]]; then
ddebug "Don't strip binaries"
return 0
fi
ddebug "Strip binaries"
find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | \
xargs strip --strip-unneeded |& \
grep -vi 'file format not recognized' | \
ddebug
}
create_rc_local() {
mkdir -p $initdir/etc/rc.d
cat >$initdir/etc/rc.d/rc.local <<EOF
#!/usr/bin/env bash
exit 0
EOF
chmod 0755 $initdir/etc/rc.d/rc.local
}
install_execs() {
ddebug "install any Execs from the service files"
(
export PKG_CONFIG_PATH=$BUILD_DIR/src/core/
systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
| sort -u | while read i; do
# some {rc,halt}.local scripts and programs are okay to not exist, the rest should
# also, plymouth is pulled in by rescue.service, but even there the exit code
# is ignored; as it's not present on some distros, don't fail if it doesn't exist
dinfo "Attempting to install $i"
inst $i || [ "${i%.local}" != "$i" ] || [ "${i%systemd-update-done}" != "$i" ] || [ "${i##*/}" == "plymouth" ]
done
)
}
generate_module_dependencies() {
if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
! depmod -a -b "$initdir" $KERNEL_VER; then
dfatal "\"depmod -a $KERNEL_VER\" failed."
exit 1
fi
}
install_depmod_files() {
inst /lib/modules/$KERNEL_VER/modules.order
inst /lib/modules/$KERNEL_VER/modules.builtin
}
install_plymouth() {
# install plymouth, if found... else remove plymouth service files
# if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
# PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
# /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
# dracut_install plymouth plymouthd
# else
rm -f $initdir/{usr/lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,etc}/systemd/system/*/plymouth*
# fi
}
install_ld_so_conf() {
cp -a /etc/ld.so.conf* $initdir/etc
ldconfig -r "$initdir"
}
install_testuser() {
# create unprivileged user for user manager tests
mkdir -p $initdir/etc/sysusers.d
cat >$initdir/etc/sysusers.d/testuser.conf <<EOF
u testuser 4711 "Test User" /home/testuser
EOF
mkdir -p $initdir/home/testuser -m 0700
chown 4711:4711 $initdir/home/testuser
}
install_config_files() {
inst /etc/sysconfig/init || :
inst /etc/passwd
inst /etc/shadow
inst /etc/login.defs
inst /etc/group
inst /etc/shells
inst /etc/nsswitch.conf
inst /etc/pam.conf || :
inst /etc/os-release
inst /etc/localtime
# we want an empty environment
> $initdir/etc/environment
> $initdir/etc/machine-id
# set the hostname
echo systemd-testsuite > $initdir/etc/hostname
# let's set up just one image with the traditional verbose output
if [ ${IMAGE_NAME} != "basic" ]; then
mkdir -p $initdir/etc/systemd/system.conf.d
echo -e '[Manager]\nStatusUnitFormat=name' >$initdir/etc/systemd/system.conf.d/status.conf
fi
}
install_basic_tools() {
dracut_install "${BASICTOOLS[@]}"
dracut_install -o sushell
# in Debian ldconfig is just a shell script wrapper around ldconfig.real
dracut_install -o ldconfig.real
}
install_debug_tools() {
dracut_install "${DEBUGTOOLS[@]}"
if [[ $INTERACTIVE_DEBUG ]]; then
# Set default TERM from vt220 to linux, so at least basic key shortcuts work
local _getty_override="$initdir/etc/systemd/system/serial-getty@.service.d"
mkdir -p "$_getty_override"
echo -e "[Service]\nEnvironment=TERM=linux" > "$_getty_override/default-TERM.conf"
cat > "$initdir/etc/motd" << EOF
To adjust the terminal size use:
export COLUMNS=xx
export LINES=yy
or
stty cols xx rows yy
EOF
fi
}
install_libnss() {
# install libnss_files for login
NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
dracut_install $NSS_LIBS
}
install_dbus() {
inst $ROOTLIBDIR/system/dbus.socket
# Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
if [ -f $ROOTLIBDIR/system/dbus-broker.service ]; then
inst $ROOTLIBDIR/system/dbus-broker.service
inst_symlink /etc/systemd/system/dbus.service
inst /usr/bin/dbus-broker
inst /usr/bin/dbus-broker-launch
elif [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
# Fedora rawhide replaced dbus.service with dbus-daemon.service
inst $ROOTLIBDIR/system/dbus-daemon.service
# Alias symlink
inst_symlink /etc/systemd/system/dbus.service
else
inst $ROOTLIBDIR/system/dbus.service
fi
find \
/etc/dbus-1 /usr/share/dbus-1 -xtype f \
| while read file; do
inst $file
done
# setup policy for Type=dbus test
mkdir -p $initdir/etc/dbus-1/system.d
cat > $initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf <<EOF
<?xml version="1.0"?>
<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
"http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
<policy user="root">
<allow own="systemd.test.ExecStopPost"/>
</policy>
</busconfig>
EOF
}
install_user_dbus() {
inst $ROOTLIBDIR/user/dbus.socket
inst_symlink /usr/lib/systemd/user/sockets.target.wants/dbus.socket || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
# Append the After= dependency on dbus in case it isn't already set up
mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
cat <<EOF >"$initdir/etc/systemd/system/user@.service.d/dbus.conf"
[Unit]
After=dbus.service
EOF
# Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
if [ -f $ROOTLIBDIR/user/dbus-broker.service ]; then
inst $ROOTLIBDIR/user/dbus-broker.service
inst_symlink /etc/systemd/user/dbus.service
elif [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
# Fedora rawhide replaced dbus.service with dbus-daemon.service
inst $ROOTLIBDIR/user/dbus-daemon.service
# Alias symlink
inst_symlink /etc/systemd/user/dbus.service
else
inst $ROOTLIBDIR/user/dbus.service
fi
}
install_pam() {
(
if [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null; then
find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
else
find /lib*/security -xtype f
fi
find /etc/pam.d /etc/security -xtype f
) | while read file; do
inst $file
done
# pam_unix depends on unix_chkpwd.
# see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
dracut_install -o unix_chkpwd
[[ "$LOOKS_LIKE_DEBIAN" ]] &&
cp /etc/pam.d/systemd-user $initdir/etc/pam.d/
# set empty root password for easy debugging
sed -i 's/^root:x:/root::/' $initdir/etc/passwd
}
install_keymaps() {
# The first three paths may be deprecated.
# It seems now the last two paths are used by many distributions.
for i in \
/usr/lib/kbd/keymaps/include/* \
/usr/lib/kbd/keymaps/i386/include/* \
/usr/lib/kbd/keymaps/i386/qwerty/us.* \
/usr/lib/kbd/keymaps/legacy/include/* \
/usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
[[ -f $i ]] || continue
inst $i
done
# When it takes any argument, then install more keymaps.
if [[ -n $1 ]]; then
for i in \
/usr/lib/kbd/keymaps/i386/*/* \
/usr/lib/kbd/keymaps/legacy/i386/*/*; do
[[ -f $i ]] || continue
inst $i
done
fi
}
install_zoneinfo() {
inst_any /usr/share/zoneinfo/Asia/Seoul
inst_any /usr/share/zoneinfo/Asia/Vladivostok
inst_any /usr/share/zoneinfo/Australia/Sydney
inst_any /usr/share/zoneinfo/Europe/Berlin
inst_any /usr/share/zoneinfo/Europe/Kiev
inst_any /usr/share/zoneinfo/Pacific/Auckland
inst_any /usr/share/zoneinfo/Pacific/Honolulu
inst_any /usr/share/zoneinfo/CET
inst_any /usr/share/zoneinfo/EET
inst_any /usr/share/zoneinfo/UTC
}
install_fonts() {
for i in \
/usr/lib/kbd/consolefonts/eurlatgr* \
/usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
[[ -f $i ]] || continue
inst $i
done
}
install_terminfo() {
for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
[ -f ${_terminfodir}/l/linux ] && break
done
dracut_install -o ${_terminfodir}/l/linux
}
has_user_dbus_socket() {
if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
return 0
else
echo "Per-user instances are not supported. Skipping..."
return 1
fi
}
setup_nspawn_root() {
if [ -z "${initdir}" ]; then
dfatal "\$initdir not defined"
exit 1
fi
rm -rf "$TESTDIR/unprivileged-nspawn-root"
if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
cp -ar $initdir $TESTDIR/unprivileged-nspawn-root
fi
}
setup_basic_dirs() {
mkdir -p $initdir/run
mkdir -p $initdir/etc/systemd/system
mkdir -p $initdir/var/log/journal
for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log dev proc sys sysroot root run run/lock run/initramfs; do
if [ -L "/$d" ]; then
inst_symlink "/$d"
else
inst_dir "/$d"
fi
done
ln -sfn /run "$initdir/var/run"
ln -sfn /run/lock "$initdir/var/lock"
}
mask_supporting_services() {
# mask some services that we do not want to run in these tests
ln -fs /dev/null $initdir/etc/systemd/system/systemd-hwdb-update.service
ln -fs /dev/null $initdir/etc/systemd/system/systemd-journal-catalog-update.service
ln -fs /dev/null $initdir/etc/systemd/system/systemd-networkd.service
ln -fs /dev/null $initdir/etc/systemd/system/systemd-networkd.socket
ln -fs /dev/null $initdir/etc/systemd/system/systemd-resolved.service
}
inst_libs() {
local _bin=$1
local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
local _file _line
LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
[[ $_line = 'not a dynamic executable' ]] && break
if [[ $_line =~ $_so_regex ]]; then
_file=${BASH_REMATCH[1]}
[[ -e ${initdir}/$_file ]] && continue
inst_library "$_file"
continue
fi
if [[ $_line =~ not\ found ]]; then
dfatal "Missing a shared library required by $_bin."
dfatal "Run \"ldd $_bin\" to find out what it is."
dfatal "$_line"
dfatal "dracut cannot create an initrd."
exit 1
fi
done
}
import_testdir() {
# make sure we don't get a stale LOOPDEV value from old times
__LOOPDEV=$LOOPDEV
[[ -e $STATEFILE ]] && . $STATEFILE
LOOPDEV=$__LOOPDEV
if [[ ! -d "$TESTDIR" ]]; then
if [[ -z "$TESTDIR" ]]; then
TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
else
mkdir -p "$TESTDIR"
fi
cat >$STATEFILE<<EOF
TESTDIR="$TESTDIR"
EOF
export TESTDIR
fi
}
import_initdir() {
initdir=$TESTDIR/root
mkdir -p $initdir
export initdir
}
## @brief Converts numeric logging level to the first letter of level name.
#
# @param lvl Numeric logging level in range from 1 to 6.
# @retval 1 if @a lvl is out of range.
# @retval 0 if @a lvl is correct.
# @result Echoes first letter of level name.
_lvl2char() {
case "$1" in
1) echo F;;
2) echo E;;
3) echo W;;
4) echo I;;
5) echo D;;
6) echo T;;
*) return 1;;
esac
}
## @brief Internal helper function for _do_dlog()
#
# @param lvl Numeric logging level.
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
#
# @note This function is not supposed to be called manually. Please use
# dtrace(), ddebug(), or others instead which wrap this one.
#
# This function calls _do_dlog() either with parameter msg, or if
# none is given, it will read standard input and will use every line as
# a message.
#
# This enables:
# dwarn "This is a warning"
# echo "This is a warning" | dwarn
LOG_LEVEL=${LOG_LEVEL:-4}
dlog() {
[ -z "$LOG_LEVEL" ] && return 0
[ $1 -le $LOG_LEVEL ] || return 0
local lvl="$1"; shift
local lvlc=$(_lvl2char "$lvl") || return 0
if [ $# -ge 1 ]; then
echo "$lvlc: $*"
else
while read line; do
echo "$lvlc: " "$line"
done
fi
}
## @brief Logs message at TRACE level (6)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dtrace() {
set +x
dlog 6 "$@"
[ -n "$debug" ] && set -x || :
}
## @brief Logs message at DEBUG level (5)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
ddebug() {
# set +x
dlog 5 "$@"
# [ -n "$debug" ] && set -x || :
}
## @brief Logs message at INFO level (4)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dinfo() {
set +x
dlog 4 "$@"
[ -n "$debug" ] && set -x || :
}
## @brief Logs message at WARN level (3)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dwarn() {
set +x
dlog 3 "$@"
[ -n "$debug" ] && set -x || :
}
## @brief Logs message at ERROR level (2)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
derror() {
# set +x
dlog 2 "$@"
# [ -n "$debug" ] && set -x || :
}
## @brief Logs message at FATAL level (1)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dfatal() {
set +x
dlog 1 "$@"
[ -n "$debug" ] && set -x || :
}
# Generic substring function. If $2 is in $1, return 0.
strstr() { [ "${1#*$2*}" != "$1" ]; }
# normalize_path <path>
# Prints the normalized path, where it removes any duplicated
# and trailing slashes.
# Example:
# $ normalize_path ///test/test//
# /test/test
normalize_path() {
shopt -q -s extglob
set -- "${1//+(\/)//}"
shopt -q -u extglob
echo "${1%/}"
}
# convert_abs_rel <from> <to>
# Prints the relative path, when creating a symlink to <to> from <from>.
# Example:
# $ convert_abs_rel /usr/bin/test /bin/test-2
# ../../bin/test-2
# $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
convert_abs_rel() {
local __current __absolute __abssize __cursize __newpath
local -i __i __level
set -- "$(normalize_path "$1")" "$(normalize_path "$2")"
# corner case #1 - self looping link
[[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
# corner case #2 - own dir link
[[ "${1%/*}" == "$2" ]] && { echo "."; return; }
IFS="/" __current=($1)
IFS="/" __absolute=($2)
__abssize=${#__absolute[@]}
__cursize=${#__current[@]}
while [[ ${__absolute[__level]} == ${__current[__level]} ]]
do
(( __level++ ))
if (( __level > __abssize || __level > __cursize ))
then
break
fi
done
for ((__i = __level; __i < __cursize-1; __i++))
do
if ((__i > __level))
then
__newpath=$__newpath"/"
fi
__newpath=$__newpath".."
done
for ((__i = __level; __i < __abssize; __i++))
do
if [[ -n $__newpath ]]
then
__newpath=$__newpath"/"
fi
__newpath=$__newpath${__absolute[__i]}
done
echo "$__newpath"
}
# Install a directory, keeping symlinks as on the original system.
# Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
# will create ${initdir}/lib64, ${initdir}/lib64/file,
# and a symlink ${initdir}/lib -> lib64.
inst_dir() {
[[ -e ${initdir}/"$1" ]] && return 0 # already there
local _dir="$1" _part="${1%/*}" _file
while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
_dir="$_part $_dir"
_part=${_part%/*}
done
# iterate over parent directories
for _file in $_dir; do
[[ -e "${initdir}/$_file" ]] && continue
if [[ -L $_file ]]; then
inst_symlink "$_file"
else
# create directory
mkdir -m 0755 -p "${initdir}/$_file" || return 1
[[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
chmod u+w "${initdir}/$_file"
fi
done
}
# $1 = file to copy to ramdisk
# $2 (optional) Name for the file on the ramdisk
# Location of the image dir is assumed to be $initdir
# We never overwrite the target if it exists.
inst_simple() {
[[ -f "$1" ]] || return 1
strstr "$1" "/" || return 1
local _src=$1 target="${2:-$1}"
if ! [[ -d ${initdir}/$target ]]; then
[[ -e ${initdir}/$target ]] && return 0
[[ -L ${initdir}/$target ]] && return 0
[[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
fi
# install checksum files also
if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
fi
ddebug "Installing $_src"
cp --sparse=always -pfL "$_src" "${initdir}/$target"
}
# find symlinks linked to given library file
# $1 = library file
# Function searches for symlinks by stripping version numbers appended to
# library filename, checks if it points to the same target and finally
# prints the list of symlinks to stdout.
#
# Example:
# rev_lib_symlinks libfoo.so.8.1
# output: libfoo.so.8 libfoo.so
# (Only if libfoo.so.8 and libfoo.so exists on host system.)
rev_lib_symlinks() {
[[ ! $1 ]] && return 0
local fn="$1" orig="$(readlink -f "$1")" links=''
[[ ${fn} =~ .*\.so\..* ]] || return 1
until [[ ${fn##*.} == so ]]; do
fn="${fn%.*}"
[[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
done
echo "${links}"
}
# Same as above, but specialized to handle dynamic libraries.
# It handles making symlinks according to how the original library
# is referenced.
inst_library() {
local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
strstr "$1" "/" || return 1
[[ -e $initdir/$_dest ]] && return 0
if [[ -L $_src ]]; then
# install checksum files also
if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
fi
_reallib=$(readlink -f "$_src")
inst_simple "$_reallib" "$_reallib"
inst_dir "${_dest%/*}"
[[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
else
inst_simple "$_src" "$_dest"
fi
# Create additional symlinks. See rev_symlinks description.
for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
[[ -e $initdir/$_symlink ]] || {
ddebug "Creating extra symlink: $_symlink"
inst_symlink $_symlink
}
done
}
# find a binary. If we were not passed the full path directly,
# search in the usual places to find the binary.
find_binary() {
if [[ -z ${1##/*} ]]; then
if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; }; then
echo $1
return 0
fi
fi
type -P $1
}
# Same as above, but specialized to install binary executables.
# Install binary executable, and all shared library dependencies, if any.
inst_binary() {
local _bin _target
# In certain cases we might attempt to install a binary which is already
# present in the test image, yet it's missing from the host system.
# In such cases, let's check if the binary indeed exists in the image
# before doing any other chcecks. If it does, immediately return with
# success.
[[ $# -eq 1 && -e $initdir/$1 ]] && return 0
_bin=$(find_binary "$1") || return 1
_target=${2:-$_bin}
[[ -e $initdir/$_target ]] && return 0
[[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
local _file _line
local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
# I love bash!
LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
[[ $_line = 'not a dynamic executable' ]] && break
if [[ $_line =~ $_so_regex ]]; then
_file=${BASH_REMATCH[1]}
[[ -e ${initdir}/$_file ]] && continue
inst_library "$_file"
continue
fi
if [[ $_line =~ not\ found ]]; then
dfatal "Missing a shared library required by $_bin."
dfatal "Run \"ldd $_bin\" to find out what it is."
dfatal "$_line"
dfatal "dracut cannot create an initrd."
exit 1
fi
done
inst_simple "$_bin" "$_target"
}
# same as above, except for shell scripts.
# If your shell script does not start with shebang, it is not a shell script.
inst_script() {
local _bin
_bin=$(find_binary "$1") || return 1
shift
local _line _shebang_regex
read -r -n 80 _line <"$_bin"
# If debug is set, clean unprintable chars to prevent messing up the term
[[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
_shebang_regex='(#! *)(/[^ ]+).*'
[[ $_line =~ $_shebang_regex ]] || return 1
inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
}
# same as above, but specialized for symlinks
inst_symlink() {
local _src=$1 _target=${2:-$1} _realsrc
strstr "$1" "/" || return 1
[[ -L $1 ]] || return 1
[[ -L $initdir/$_target ]] && return 0
_realsrc=$(readlink -f "$_src")
if ! [[ -e $initdir/$_realsrc ]]; then
if [[ -d $_realsrc ]]; then
inst_dir "$_realsrc"
else
inst "$_realsrc"
fi
fi
[[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
[[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
}
# attempt to install any programs specified in a udev rule
inst_rule_programs() {
local _prog _bin
if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
if [ -x /lib/udev/$_prog ]; then
_bin=/lib/udev/$_prog
else
_bin=$(find_binary "$_prog") || {
dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
continue;
}
fi
#dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
dracut_install "$_bin"
done
fi
}
# udev rules always get installed in the same place, so
# create a function to install them to make life simpler.
inst_rules() {
local _target=/etc/udev/rules.d _rule _found
inst_dir "/lib/udev/rules.d"
inst_dir "$_target"
for _rule in "$@"; do
if [ "${rule#/}" = "$rule" ]; then
for r in /lib/udev/rules.d /etc/udev/rules.d; do
if [[ -f $r/$_rule ]]; then
_found="$r/$_rule"
inst_simple "$_found"
inst_rule_programs "$_found"
fi
done
fi
for r in '' ./ $dracutbasedir/rules.d/; do
if [[ -f ${r}$_rule ]]; then
_found="${r}$_rule"
inst_simple "$_found" "$_target/${_found##*/}"
inst_rule_programs "$_found"
fi
done
[[ $_found ]] || dinfo "Skipping udev rule: $_rule"
_found=
done
}
# general purpose installation function
# Same args as above.
inst() {
local _x
case $# in
1) ;;
2) [[ ! $initdir && -d $2 ]] && export initdir=$2
[[ $initdir = $2 ]] && set $1;;
3) [[ -z $initdir ]] && export initdir=$2
set $1 $3;;
*) dfatal "inst only takes 1 or 2 or 3 arguments"
exit 1;;
esac
for _x in inst_symlink inst_script inst_binary inst_simple; do
$_x "$@" && return 0
done
return 1
}
# install any of listed files
#
# If first argument is '-d' and second some destination path, first accessible
# source is installed into this path, otherwise it will installed in the same
# path as source. If none of listed files was installed, function return 1.
# On first successful installation it returns with 0 status.
#
# Example:
#
# inst_any -d /bin/foo /bin/bar /bin/baz
#
# Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
# initramfs.
inst_any() {
local to f
[[ $1 = '-d' ]] && to="$2" && shift 2
for f in "$@"; do
if [[ -e $f ]]; then
[[ $to ]] && inst "$f" "$to" && return 0
inst "$f" && return 0
fi
done
return 1
}
# dracut_install [-o ] <file> [<file> ... ]
# Install <file> to the initramfs image
# -o optionally install the <file> and don't fail, if it is not there
dracut_install() {
local _optional=no
if [[ $1 = '-o' ]]; then
_optional=yes
shift
fi
while (($# > 0)); do
if ! inst "$1" ; then
if [[ $_optional = yes ]]; then
dinfo "Skipping program $1 as it cannot be found and is" \
"flagged to be optional"
else
dfatal "Failed to install $1"
exit 1
fi
fi
shift
done
}
# Install a single kernel module along with any firmware it may require.
# $1 = full path to kernel module to install
install_kmod_with_fw() {
# no need to go further if the module is already installed
[[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
&& return 0
[[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0
if [[ $omit_drivers ]]; then
local _kmod=${1##*/}
_kmod=${_kmod%.ko}
_kmod=${_kmod/-/_}
if [[ "$_kmod" =~ $omit_drivers ]]; then
dinfo "Omitting driver $_kmod"
return 1
fi
if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
dinfo "Omitting driver $_kmod"
return 1
fi
fi
[ -d "$initdir/.kernelmodseen" ] && \
> "$initdir/.kernelmodseen/${1##*/}"
inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
|| return $?
local _modname=${1##*/} _fwdir _found _fw
_modname=${_modname%.ko*}
for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
_found=''
for _fwdir in $fw_dir; do
if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
_found=yes
fi
done
if [[ $_found != yes ]]; then
if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
"\"${_modname}.ko\""
else
dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
"\"${_modname}.ko\""
fi
fi
done
return 0
}
# Do something with all the dependencies of a kernel module.
# Note that kernel modules depend on themselves using the technique we use
# $1 = function to call for each dependency we find
# It will be passed the full path to the found kernel module
# $2 = module to get dependencies for
# rest of args = arguments to modprobe
# _fderr specifies FD passed from surrounding scope
for_each_kmod_dep() {
local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
shift 2
modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
while read _cmd _modpath _options; do
[[ $_cmd = insmod ]] || continue
$_func ${_modpath} || exit $?
_found=1
done
[[ $_found -eq 0 ]] && exit 1
exit 0
)
}
# filter kernel modules to install certain modules that meet specific
# requirements.
# $1 = search only in subdirectory of /kernel/$1
# $2 = function to call with module name to filter.
# This function will be passed the full path to the module to test.
# The behavior of this function can vary depending on whether $hostonly is set.
# If it is, we will only look at modules that are already in memory.
# If it is not, we will look at all kernel modules
# This function returns the full filenames of modules that match $1
filter_kernel_modules_by_path () (
local _modname _filtercmd
if ! [[ $hostonly ]]; then
_filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
_filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
_filtercmd+=' -o -name "*.ko.xz"'
_filtercmd+=' 2>/dev/null'
else
_filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
_filtercmd+='-k $KERNEL_VER 2>/dev/null'
fi
for _modname in $(eval $_filtercmd); do
case $_modname in
*.ko) "$2" "$_modname" && echo "$_modname";;
*.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
$2 $initdir/$$.ko && echo "$_modname"
rm -f $initdir/$$.ko
;;
*.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
$2 $initdir/$$.ko && echo "$_modname"
rm -f $initdir/$$.ko
;;
esac
done
)
find_kernel_modules_by_path () (
if ! [[ $hostonly ]]; then
find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
-name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
else
cut -d " " -f 1 </proc/modules \
| xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
fi
)
filter_kernel_modules () {
filter_kernel_modules_by_path drivers "$1"
}
find_kernel_modules () {
find_kernel_modules_by_path drivers
}
# instmods [-c] <kernel module> [<kernel module> ... ]
# instmods [-c] <kernel subsystem>
# install kernel modules along with all their dependencies.
# <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
instmods() {
[[ $no_kernel = yes ]] && return
# called [sub]functions inherit _fderr
local _fderr=9
local _check=no
if [[ $1 = '-c' ]]; then
_check=yes
shift
fi
function inst1mod() {
local _ret=0 _mod="$1"
case $_mod in
=*)
if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
( [[ "$_mpargs" ]] && echo $_mpargs
cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
| instmods
else
( [[ "$_mpargs" ]] && echo $_mpargs
find "$KERNEL_MODS" -path "*/${_mod#=}/*" -type f -printf '%f\n' ) \
| instmods
fi
;;
--*) _mpargs+=" $_mod" ;;
i2o_scsi) return ;; # Do not load this diagnostic-only module
*)
_mod=${_mod##*/}
# if we are already installed, skip this module and go on
# to the next one.
[[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return
if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
return
fi
# If we are building a host-specific initramfs and this
# module is not already loaded, move on to the next one.
[[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
&& ! echo $add_drivers | grep -qe "\<${_mod}\>" \
&& return
# We use '-d' option in modprobe only if modules prefix path
# differs from default '/'. This allows us to use Dracut with
# old version of modprobe which doesn't have '-d' option.
local _moddirname=${KERNEL_MODS%%/lib/modules/*}
[[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"
# ok, load the module, all its dependencies, and any firmware
# it may require
for_each_kmod_dep install_kmod_with_fw $_mod \
--set-version $KERNEL_VER ${_moddirname} $_mpargs
((_ret+=$?))
;;
esac
return $_ret
}
function instmods_1() {
local _mod _mpargs
if (($# == 0)); then # filenames from stdin
while read _mod; do
inst1mod "${_mod%.ko*}" || {
if [ "$_check" = "yes" ]; then
dfatal "Failed to install $_mod"
return 1
fi
}
done
fi
while (($# > 0)); do # filenames as arguments
inst1mod ${1%.ko*} || {
if [ "$_check" = "yes" ]; then
dfatal "Failed to install $1"
return 1
fi
}
shift
done
return 0
}
local _ret _filter_not_found='FATAL: Module .* not found.'
set -o pipefail
# Capture all stderr from modprobe to _fderr. We could use {var}>...
# redirections, but that would make dracut require bash4 at least.
eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
| while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
_ret=$?
set +o pipefail
return $_ret
}
setup_suse() {
ln -fs ../usr/bin/systemctl $initdir/bin/
ln -fs ../usr/lib/systemd $initdir/lib/
inst_simple "/usr/lib/systemd/system/haveged.service"
}
_umount_dir() {
if mountpoint -q $1; then
ddebug "umount $1"
umount $1
fi
}
# can be overridden in specific test
test_setup_cleanup() {
cleanup_initdir
}
_test_cleanup() {
# (post-test) cleanup should always ignore failure and cleanup as much as possible
(
set +e
_umount_dir $initdir
rm -vf "${IMAGESTATEDIR}/${IMAGE_NAME}.img"
rm -vfr "$TESTDIR"
rm -vf "$STATEFILE"
) || :
}
# can be overridden in specific test
test_cleanup() {
_test_cleanup
}
test_cleanup_again() {
[ -n "$TESTDIR" ] || return
rm -rf "$TESTDIR/unprivileged-nspawn-root"
_umount_dir $initdir
}
test_create_image() {
create_empty_image_rootdir
# Create what will eventually be our root filesystem onto an overlay
(
LOG_LEVEL=5
setup_basic_environment
mask_supporting_services
)
}
test_setup() {
if [ ${TEST_REQUIRE_INSTALL_TESTS} -ne 0 ] && \
type -P meson >/dev/null && \
[[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
exit 1
fi
image="${TESTDIR}/${IMAGE_NAME}.img"
public="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
if [ -e "$image" ]; then
echo "Reusing existing image $PWD/$image → $(realpath $image)"
mount_initdir
elif [ -e "$public" ]; then
echo "Reusing existing cached image $PWD/$public → $(realpath $public)"
ln -s "$(realpath $public)" "$image"
mount_initdir
else
test_create_image
fi
setup_nspawn_root
}
test_run() {
mount_initdir
if [ -z "$TEST_NO_QEMU" ]; then
if run_qemu "$1"; then
check_result_qemu || { echo "QEMU test failed"; return 1; }
else
dwarn "can't run QEMU, skipping"
fi
fi
if [ -z "$TEST_NO_NSPAWN" ]; then
mount_initdir
if run_nspawn "$initdir" "$1"; then
check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
else
dwarn "can't run systemd-nspawn, skipping"
fi
if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
dir="$TESTDIR/unprivileged-nspawn-root"
if NSPAWN_ARGUMENTS="-U --private-network $NSPAWN_ARGUMENTS" run_nspawn "$dir" "$1"; then
check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
else
dwarn "can't run systemd-nspawn, skipping"
fi
fi
fi
return 0
}
do_test() {
if [[ $UID != "0" ]]; then
echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
exit 0
fi
# Detect lib paths
[[ $libdir ]] || for libdir in /lib64 /lib; do
[[ -d $libdir ]] && libdirs+=" $libdir" && break
done
[[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
[[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
done
mkdir -p "$STATEDIR"
import_testdir
import_initdir
testname="$(basename $PWD)"
while (($# > 0)); do
case $1 in
--run)
echo "${testname} RUN: $TEST_DESCRIPTION"
test_run "$2"
ret=$?
if (( $ret == 0 )); then
echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
else
echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
fi
exit $ret;;
--setup)
echo "${testname} SETUP: $TEST_DESCRIPTION"
test_setup
test_setup_cleanup
;;
--clean)
echo "${testname} CLEANUP: $TEST_DESCRIPTION"
test_cleanup
;;
--clean-again)
echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
test_cleanup_again
;;
--all)
ret=0
echo -n "${testname}: $TEST_DESCRIPTION "
(
test_setup
test_setup_cleanup
test_run "$2"
) </dev/null >"$TESTLOG" 2>&1 || ret=$?
test_cleanup
if [ $ret -eq 0 ]; then
rm "$TESTLOG"
echo "[OK]"
else
echo "[FAILED]"
echo "see $TESTLOG"
fi
exit $ret;;
*) break ;;
esac
shift
done
}
View Git Blame Copy Permalink