systemd-stable/test/test-functions

#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin
export PATH

LOOKS_LIKE_DEBIAN=$(source /etc/os-release && [[ "$ID" = "debian" || "$ID_LIKE" = "debian" ]] && echo yes)
KERNEL_VER=${KERNEL_VER-$(uname -r)}
KERNEL_MODS="/lib/modules/$KERNEL_VER/"

if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
    echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
    ROOTLIBDIR=/usr/lib/systemd
fi

BASICTOOLS="sh bash setsid loadkeys setfont login sulogin gzip sleep echo mount umount cryptsetup date dmsetup modprobe"
DEBUGTOOLS="df free ls stty cat ps ln ip route dmesg dhclient mkdir cp ping dhclient strace less grep id tty touch du sort hostname"

function find_qemu_bin() {
    # SUSE and Red Hat call the binary qemu-kvm
    # Debian and Gentoo call it kvm
    [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)

    [ "$ARCH" ] || ARCH=$(uname -m)
    case $ARCH in
    x86_64)
        # QEMU's own build system calls it qemu-system-x86_64
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
        ;;
    i*86)
        # new i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)

        # i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
        ;;
    esac

    if [ ! -e "$QEMU_BIN" ]; then
        echo "Could not find a suitable QEMU binary" >&2
        return 1
    fi
}

run_qemu() {
    if [ -f /etc/machine-id ]; then
        read MACHINE_ID < /etc/machine-id
        [ -z "$INITRD" ] && [ -e "/boot/$MACHINE_ID/$KERNEL_VER/initrd" ] \
            && INITRD="/boot/$MACHINE_ID/$KERNEL_VER/initrd"
        [ -z "$KERNEL_BIN" ] && [ -e "/boot/$MACHINE_ID/$KERNEL_VER/linux" ] \
            && KERNEL_BIN="/boot/$MACHINE_ID/$KERNEL_VER/linux"
    fi

    [ "$KERNEL_BIN" ] || KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
    [ "$INITRD" ]     || INITRD=/boot/initramfs-${KERNEL_VER}.img
    [ "$QEMU_SMP" ]   || QEMU_SMP=1

    find_qemu_bin || return 1

    KERNEL_APPEND="root=/dev/sda1 \
systemd.log_level=debug \
raid=noautodetect \
loglevel=2 \
init=$ROOTLIBDIR/systemd \
ro \
console=ttyS0 \
selinux=0 \
$KERNEL_APPEND \
"

    QEMU_OPTIONS="-machine accel=kvm:tcg \
-smp $QEMU_SMP \
-net none \
-m 512M \
-nographic \
-kernel $KERNEL_BIN \
"

    if [ "$INITRD" ]; then
        QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
    fi

    ( set -x
      $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND" $TESTDIR/rootdisk.img ) || return 1
}

run_nspawn() {
    set -x
    ../../systemd-nspawn --boot --directory=$TESTDIR/nspawn-root $ROOTLIBDIR/systemd $KERNEL_APPEND
}

setup_basic_environment() {
    # create the basic filesystem layout
    setup_basic_dirs

    install_systemd
    install_missing_libraries
    install_config_files
    create_rc_local
    install_basic_tools
    install_libnss
    install_pam
    install_dbus
    install_fonts
    install_keymaps
    install_terminfo
    install_execs
    install_plymouth
    install_debug_tools
    install_ld_so_conf
    strip_binaries
    install_depmod_files
    generate_module_dependencies
    # softlink mtab
    ln -fs /proc/self/mounts $initdir/etc/mtab
}

install_dmevent() {
    instmods dm_crypt =crypto
    type -P dmeventd >/dev/null && dracut_install dmeventd
    inst_libdir_file "libdevmapper-event.so*"
    inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
}

install_systemd() {
    # install compiled files
    (cd $TEST_BASE_DIR/..; set -x; make DESTDIR=$initdir install)
    # remove unneeded documentation
    rm -fr $initdir/usr/share/{man,doc}
    # we strip binaries since debug symbols increase binaries size a lot
    # and it could fill the available space
    strip_binaries
}

install_missing_libraries() {
    # install possible missing libraries
    for i in $initdir/{sbin,bin}/* $initdir/lib/systemd/*; do
        inst_libs $i
    done
}

create_empty_image() {
    rm -f "$TESTDIR/rootdisk.img"
    # Create the blank file to use as a root filesystem
    dd if=/dev/null of="$TESTDIR/rootdisk.img" bs=1M seek=300
    LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
    [ -b "$LOOPDEV" ] || return 1
    echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
    sfdisk "$LOOPDEV" <<EOF
,290M
,
EOF

    mkfs.ext3 -L systemd "${LOOPDEV}p1"
}

check_result_nspawn() {
    ret=1
    [[ -e $TESTDIR/nspawn-root/testok ]] && ret=0
    [[ -f $TESTDIR/nspawn-root/failed ]] && cp -a $TESTDIR/nspawn-root/failed $TESTDIR
    cp -a $TESTDIR/nspawn-root/var/log/journal $TESTDIR
    [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
    ls -l $TESTDIR/journal/*/*.journal
    test -s $TESTDIR/failed && ret=$(($ret+1))
    return $ret
}

strip_binaries() {
    ddebug "Strip binaries"
    find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | xargs strip --strip-unneeded | ddebug
}

create_rc_local() {
    mkdir -p $initdir/etc/rc.d
    cat >$initdir/etc/rc.d/rc.local <<EOF
#!/bin/bash
exit 0
EOF
    chmod 0755 $initdir/etc/rc.d/rc.local
}

install_execs() {
    ddebug "install any Execs from the service files"
    (
    export PKG_CONFIG_PATH=$TEST_BASE_DIR/../src/core/
    systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
    systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
    egrep -ho '^Exec[^ ]*=[^ ]+' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
         | while read i; do
         i=${i##Exec*=}; i=${i##-}
         inst $i
     done
    )
}

generate_module_dependencies() {
    if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
        ! depmod -a -b "$initdir" $KERNEL_VER; then
            dfatal "\"depmod -a $KERNEL_VER\" failed."
            exit 1
    fi
}

install_depmod_files() {
    inst /lib/modules/$KERNEL_VER/modules.order
    inst /lib/modules/$KERNEL_VER/modules.builtin
}

install_plymouth() {
    # install plymouth, if found... else remove plymouth service files
    # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
    #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
    #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
    #         dracut_install plymouth plymouthd
    # else
        rm -f $initdir/{usr/lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,etc}/systemd/system/*/plymouth*
    # fi
}

install_ld_so_conf() {
    cp -a /etc/ld.so.conf* $initdir/etc
    ldconfig -r "$initdir"
}

install_config_files() {
    inst /etc/sysconfig/init
    inst /etc/passwd
    inst /etc/shadow
    inst /etc/login.defs
    inst /etc/group
    inst /etc/shells
    inst /etc/nsswitch.conf
    inst /etc/pam.conf
    inst /etc/securetty
    inst /etc/os-release
    inst /etc/localtime
    # we want an empty environment
    > $initdir/etc/environment
    > $initdir/etc/machine-id
    # set the hostname
    echo systemd-testsuite > $initdir/etc/hostname
    # fstab
    cat >$initdir/etc/fstab <<EOF
LABEL=systemd           /       ext3    rw 0 1
EOF
}

install_basic_tools() {
    [[ $BASICTOOLS ]] && dracut_install $BASICTOOLS
    dracut_install -o sushell
    # in Debian ldconfig is just a shell script wrapper around ldconfig.real
    dracut_install -o ldconfig.real
}

install_debug_tools() {
    [[ $DEBUGTOOLS ]] && dracut_install $DEBUGTOOLS
}

install_libnss() {
    # install libnss_files for login
    NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
    dracut_install $NSS_LIBS
}

install_dbus() {
    inst $ROOTLIBDIR/system/dbus.socket
    inst $ROOTLIBDIR/system/dbus.service

    find \
        /etc/dbus-1 /usr/share/dbus-1 -xtype f \
        | while read file; do
        inst $file
    done
}

install_pam() {
    (
    [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null && find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
    find \
        /etc/pam.d \
        /etc/security \
        /lib64/security \
        /lib/security -xtype f \
    ) | while read file; do
        inst $file
    done

    [[ "$LOOKS_LIKE_DEBIAN" ]] &&
        cp /etc/pam.d/systemd-user $initdir/etc/pam.d/
}

install_keymaps() {
    for i in \
        /usr/lib/kbd/keymaps/include/* \
        /usr/lib/kbd/keymaps/i386/include/* \
        /usr/lib/kbd/keymaps/i386/qwerty/us.*; do
            [[ -f $i ]] || continue
            inst $i
    done
}

install_fonts() {
    for i in \
        /usr/lib/kbd/consolefonts/eurlatgr* \
        /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
            [[ -f $i ]] || continue
            inst $i
    done
}

install_terminfo() {
    for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
        [ -f ${_terminfodir}/l/linux ] && break
    done
    dracut_install -o ${_terminfodir}/l/linux
}

setup_testsuite() {
    cp $TEST_BASE_DIR/testsuite.target $initdir/etc/systemd/system/
    cp $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/

    mkdir -p $initdir/etc/systemd/system/testsuite.target.wants
    ln -fs $TEST_BASE_DIR/testsuite.service $initdir/etc/systemd/system/testsuite.target.wants/testsuite.service
    ln -fs $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/testsuite.target.wants/end.service

    # make the testsuite the default target
    ln -fs testsuite.target $initdir/etc/systemd/system/default.target
}

setup_nspawn_root() {
    rm -fr $TESTDIR/nspawn-root
    ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
    cp -ar $initdir $TESTDIR/nspawn-root
    # we don't mount in the nspawn root
    rm -f $TESTDIR/nspawn-root/etc/fstab
}

setup_basic_dirs() {
    mkdir -p $initdir/run
    mkdir -p $initdir/etc/systemd/system
    mkdir -p $initdir/var/log/journal

    for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log dev proc sys sysroot root run run/lock run/initramfs; do
        if [ -L "/$d" ]; then
            inst_symlink "/$d"
        else
            inst_dir "/$d"
        fi
    done

    ln -sfn /run "$initdir/var/run"
    ln -sfn /run/lock "$initdir/var/lock"
}

inst_libs() {
    local _bin=$1
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    local _file _line

    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
}

import_testdir() {
    STATEFILE=".testdir"
    [[ -e $STATEFILE ]] && . $STATEFILE
    if [[ -z "$TESTDIR" ]] || [[ ! -d "$TESTDIR" ]]; then
        TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
        echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
        export TESTDIR
    fi
}

import_initdir() {
    initdir=$TESTDIR/root
    export initdir
}

## @brief Converts numeric logging level to the first letter of level name.
#
# @param lvl Numeric logging level in range from 1 to 6.
# @retval 1 if @a lvl is out of range.
# @retval 0 if @a lvl is correct.
# @result Echoes first letter of level name.
_lvl2char() {
    case "$1" in
        1) echo F;;
        2) echo E;;
        3) echo W;;
        4) echo I;;
        5) echo D;;
        6) echo T;;
        *) return 1;;
    esac
}

## @brief Internal helper function for _do_dlog()
#
# @param lvl Numeric logging level.
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
#
# @note This function is not supposed to be called manually. Please use
# dtrace(), ddebug(), or others instead which wrap this one.
#
# This function calls _do_dlog() either with parameter msg, or if
# none is given, it will read standard input and will use every line as
# a message.
#
# This enables:
# dwarn "This is a warning"
# echo "This is a warning" | dwarn
LOG_LEVEL=4

dlog() {
    [ -z "$LOG_LEVEL" ] && return 0
    [ $1 -le $LOG_LEVEL ] || return 0
    local lvl="$1"; shift
    local lvlc=$(_lvl2char "$lvl") || return 0

    if [ $# -ge 1 ]; then
        echo "$lvlc: $*"
    else
        while read line; do
            echo "$lvlc: " "$line"
        done
    fi
}

## @brief Logs message at TRACE level (6)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dtrace() {
    set +x
    dlog 6 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at DEBUG level (5)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
ddebug() {
#    set +x
    dlog 5 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at INFO level (4)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dinfo() {
    set +x
    dlog 4 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at WARN level (3)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dwarn() {
    set +x
    dlog 3 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at ERROR level (2)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
derror() {
#    set +x
    dlog 2 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at FATAL level (1)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dfatal() {
    set +x
    dlog 1 "$@"
    [ -n "$debug" ] && set -x || :
}


# Generic substring function.  If $2 is in $1, return 0.
strstr() { [ "${1#*$2*}" != "$1" ]; }

# normalize_path <path>
# Prints the normalized path, where it removes any duplicated
# and trailing slashes.
# Example:
# $ normalize_path ///test/test//
# /test/test
normalize_path() {
    shopt -q -s extglob
    set -- "${1//+(\/)//}"
    shopt -q -u extglob
    echo "${1%/}"
}

# convert_abs_rel <from> <to>
# Prints the relative path, when creating a symlink to <to> from <from>.
# Example:
# $ convert_abs_rel /usr/bin/test /bin/test-2
# ../../bin/test-2
# $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
convert_abs_rel() {
    local __current __absolute __abssize __cursize __newpath
    local -i __i __level

    set -- "$(normalize_path "$1")" "$(normalize_path "$2")"

    # corner case #1 - self looping link
    [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }

    # corner case #2 - own dir link
    [[ "${1%/*}" == "$2" ]] && { echo "."; return; }

    IFS="/" __current=($1)
    IFS="/" __absolute=($2)

    __abssize=${#__absolute[@]}
    __cursize=${#__current[@]}

    while [[ ${__absolute[__level]} == ${__current[__level]} ]]
    do
        (( __level++ ))
        if (( __level > __abssize || __level > __cursize ))
        then
            break
        fi
    done

    for ((__i = __level; __i < __cursize-1; __i++))
    do
        if ((__i > __level))
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath".."
    done

    for ((__i = __level; __i < __abssize; __i++))
    do
        if [[ -n $__newpath ]]
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath${__absolute[__i]}
    done

    echo "$__newpath"
}


# Install a directory, keeping symlinks as on the original system.
# Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
# will create ${initdir}/lib64, ${initdir}/lib64/file,
# and a symlink ${initdir}/lib -> lib64.
inst_dir() {
    [[ -e ${initdir}/"$1" ]] && return 0  # already there

    local _dir="$1" _part="${1%/*}" _file
    while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
        _dir="$_part $_dir"
        _part=${_part%/*}
    done

    # iterate over parent directories
    for _file in $_dir; do
        [[ -e "${initdir}/$_file" ]] && continue
        if [[ -L $_file ]]; then
            inst_symlink "$_file"
        else
            # create directory
            mkdir -m 0755 -p "${initdir}/$_file" || return 1
            [[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
            chmod u+w "${initdir}/$_file"
        fi
    done
}

# $1 = file to copy to ramdisk
# $2 (optional) Name for the file on the ramdisk
# Location of the image dir is assumed to be $initdir
# We never overwrite the target if it exists.
inst_simple() {
    [[ -f "$1" ]] || return 1
    strstr "$1" "/" || return 1

    local _src=$1 target="${2:-$1}"
    if ! [[ -d ${initdir}/$target ]]; then
        [[ -e ${initdir}/$target ]] && return 0
        [[ -L ${initdir}/$target ]] && return 0
        [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
    fi
    # install checksum files also
    if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
        inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
    fi
    ddebug "Installing $_src"
    cp --sparse=always -pfL "$_src" "${initdir}/$target"
}

# find symlinks linked to given library file
# $1 = library file
# Function searches for symlinks by stripping version numbers appended to
# library filename, checks if it points to the same target and finally
# prints the list of symlinks to stdout.
#
# Example:
# rev_lib_symlinks libfoo.so.8.1
# output: libfoo.so.8 libfoo.so
# (Only if libfoo.so.8 and libfoo.so exists on host system.)
rev_lib_symlinks() {
    [[ ! $1 ]] && return 0

    local fn="$1" orig="$(readlink -f "$1")" links=''

    [[ ${fn} =~ .*\.so\..* ]] || return 1

    until [[ ${fn##*.} == so ]]; do
        fn="${fn%.*}"
        [[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
    done

    echo "${links}"
}

# Same as above, but specialized to handle dynamic libraries.
# It handles making symlinks according to how the original library
# is referenced.
inst_library() {
    local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
    strstr "$1" "/" || return 1
    [[ -e $initdir/$_dest ]] && return 0
    if [[ -L $_src ]]; then
        # install checksum files also
        if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
            inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
        fi
        _reallib=$(readlink -f "$_src")
        inst_simple "$_reallib" "$_reallib"
        inst_dir "${_dest%/*}"
        [[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
        ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
    else
        inst_simple "$_src" "$_dest"
    fi

    # Create additional symlinks.  See rev_symlinks description.
    for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
        [[ ! -e $initdir/$_symlink ]] && {
            ddebug "Creating extra symlink: $_symlink"
            inst_symlink $_symlink
        }
    done
}

# find a binary.  If we were not passed the full path directly,
# search in the usual places to find the binary.
find_binary() {
    if [[ -z ${1##/*} ]]; then
        if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; };  then
            echo $1
            return 0
        fi
    fi

    type -P $1
}

# Same as above, but specialized to install binary executables.
# Install binary executable, and all shared library dependencies, if any.
inst_binary() {
    local _bin _target
    _bin=$(find_binary "$1") || return 1
    _target=${2:-$_bin}
    [[ -e $initdir/$_target ]] && return 0
    [[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
    local _file _line
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    # I love bash!
    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
    inst_simple "$_bin" "$_target"
}

# same as above, except for shell scripts.
# If your shell script does not start with shebang, it is not a shell script.
inst_script() {
    local _bin
    _bin=$(find_binary "$1") || return 1
    shift
    local _line _shebang_regex
    read -r -n 80 _line <"$_bin"
    # If debug is set, clean unprintable chars to prevent messing up the term
    [[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
    _shebang_regex='(#! *)(/[^ ]+).*'
    [[ $_line =~ $_shebang_regex ]] || return 1
    inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
}

# same as above, but specialized for symlinks
inst_symlink() {
    local _src=$1 _target=${2:-$1} _realsrc
    strstr "$1" "/" || return 1
    [[ -L $1 ]] || return 1
    [[ -L $initdir/$_target ]] && return 0
    _realsrc=$(readlink -f "$_src")
    if ! [[ -e $initdir/$_realsrc ]]; then
        if [[ -d $_realsrc ]]; then
            inst_dir "$_realsrc"
        else
            inst "$_realsrc"
        fi
    fi
    [[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
    [[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
    ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
}

# attempt to install any programs specified in a udev rule
inst_rule_programs() {
    local _prog _bin

    if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
        for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
            if [ -x /lib/udev/$_prog ]; then
                _bin=/lib/udev/$_prog
            else
                _bin=$(find_binary "$_prog") || {
                    dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
                    continue;
                }
            fi

            #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
            dracut_install "$_bin"
        done
    fi
}

# udev rules always get installed in the same place, so
# create a function to install them to make life simpler.
inst_rules() {
    local _target=/etc/udev/rules.d _rule _found

    inst_dir "/lib/udev/rules.d"
    inst_dir "$_target"
    for _rule in "$@"; do
        if [ "${rule#/}" = "$rule" ]; then
            for r in /lib/udev/rules.d /etc/udev/rules.d; do
                if [[ -f $r/$_rule ]]; then
                    _found="$r/$_rule"
                    inst_simple "$_found"
                    inst_rule_programs "$_found"
                fi
            done
        fi
        for r in '' ./ $dracutbasedir/rules.d/; do
            if [[ -f ${r}$_rule ]]; then
                _found="${r}$_rule"
                inst_simple "$_found" "$_target/${_found##*/}"
                inst_rule_programs "$_found"
            fi
        done
        [[ $_found ]] || dinfo "Skipping udev rule: $_rule"
    done
}

# general purpose installation function
# Same args as above.
inst() {
    local _x

    case $# in
        1) ;;
        2) [[ ! $initdir && -d $2 ]] && export initdir=$2
            [[ $initdir = $2 ]] && set $1;;
        3) [[ -z $initdir ]] && export initdir=$2
            set $1 $3;;
        *) dfatal "inst only takes 1 or 2 or 3 arguments"
            exit 1;;
    esac
    for _x in inst_symlink inst_script inst_binary inst_simple; do
        $_x "$@" && return 0
    done
    return 1
}

# install any of listed files
#
# If first argument is '-d' and second some destination path, first accessible
# source is installed into this path, otherwise it will installed in the same
# path as source.  If none of listed files was installed, function return 1.
# On first successful installation it returns with 0 status.
#
# Example:
#
# inst_any -d /bin/foo /bin/bar /bin/baz
#
# Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
# initramfs.
inst_any() {
    local to f

    [[ $1 = '-d' ]] && to="$2" && shift 2

    for f in "$@"; do
        if [[ -e $f ]]; then
            [[ $to ]] && inst "$f" "$to" && return 0
            inst "$f" && return 0
        fi
    done

    return 1
}

# dracut_install [-o ] <file> [<file> ... ]
# Install <file> to the initramfs image
# -o optionally install the <file> and don't fail, if it is not there
dracut_install() {
    local _optional=no
    if [[ $1 = '-o' ]]; then
        _optional=yes
        shift
    fi
    while (($# > 0)); do
        if ! inst "$1" ; then
            if [[ $_optional = yes ]]; then
                dinfo "Skipping program $1 as it cannot be found and is" \
                    "flagged to be optional"
            else
                dfatal "Failed to install $1"
                exit 1
            fi
        fi
        shift
    done
}

# Install a single kernel module along with any firmware it may require.
# $1 = full path to kernel module to install
install_kmod_with_fw() {
    # no need to go further if the module is already installed

    [[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
        && return 0

    [[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0

    if [[ $omit_drivers ]]; then
        local _kmod=${1##*/}
        _kmod=${_kmod%.ko}
        _kmod=${_kmod/-/_}
        if [[ "$_kmod" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
        if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
    fi

    [ -d "$initdir/.kernelmodseen" ] && \
        > "$initdir/.kernelmodseen/${1##*/}"

    inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
        || return $?

    local _modname=${1##*/} _fwdir _found _fw
    _modname=${_modname%.ko*}
    for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
        _found=''
        for _fwdir in $fw_dir; do
            if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
                inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
                _found=yes
            fi
        done
        if [[ $_found != yes ]]; then
            if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
                dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            else
                dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            fi
        fi
    done
    return 0
}

# Do something with all the dependencies of a kernel module.
# Note that kernel modules depend on themselves using the technique we use
# $1 = function to call for each dependency we find
#      It will be passed the full path to the found kernel module
# $2 = module to get dependencies for
# rest of args = arguments to modprobe
# _fderr specifies FD passed from surrounding scope
for_each_kmod_dep() {
    local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
    shift 2
    modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
        while read _cmd _modpath _options; do
            [[ $_cmd = insmod ]] || continue
            $_func ${_modpath} || exit $?
            _found=1
        done
        [[ $_found -eq 0 ]] && exit 1
        exit 0
    )
}

# filter kernel modules to install certain modules that meet specific
# requirements.
# $1 = search only in subdirectory of /kernel/$1
# $2 = function to call with module name to filter.
#      This function will be passed the full path to the module to test.
# The behavior of this function can vary depending on whether $hostonly is set.
# If it is, we will only look at modules that are already in memory.
# If it is not, we will look at all kernel modules
# This function returns the full filenames of modules that match $1
filter_kernel_modules_by_path () (
    local _modname _filtercmd
    if ! [[ $hostonly ]]; then
        _filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
        _filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
        _filtercmd+=' -o -name "*.ko.xz"'
        _filtercmd+=' 2>/dev/null'
    else
        _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
        _filtercmd+='-k $KERNEL_VER 2>/dev/null'
    fi
    for _modname in $(eval $_filtercmd); do
        case $_modname in
            *.ko) "$2" "$_modname" && echo "$_modname";;
            *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
            *.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
        esac
    done
)
find_kernel_modules_by_path () (
    if ! [[ $hostonly ]]; then
        find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
          -name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
    else
        cut -d " " -f 1 </proc/modules \
        | xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
    fi
)

filter_kernel_modules () {
    filter_kernel_modules_by_path  drivers  "$1"
}

find_kernel_modules () {
    find_kernel_modules_by_path  drivers
}

# instmods [-c] <kernel module> [<kernel module> ... ]
# instmods [-c] <kernel subsystem>
# install kernel modules along with all their dependencies.
# <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
instmods() {
    [[ $no_kernel = yes ]] && return
    # called [sub]functions inherit _fderr
    local _fderr=9
    local _check=no
    if [[ $1 = '-c' ]]; then
        _check=yes
        shift
    fi

    function inst1mod() {
        local _ret=0 _mod="$1"
        case $_mod in
            =*)
                if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
                    | instmods
                else
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      find "$KERNEL_MODS" -path "*/${_mod#=}/*" -printf '%f\n' ) \
                    | instmods
                fi
                ;;
            --*) _mpargs+=" $_mod" ;;
            i2o_scsi) return ;; # Do not load this diagnostic-only module
            *)
                _mod=${_mod##*/}
                # if we are already installed, skip this module and go on
                # to the next one.
                [[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return

                if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
                    dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
                    return
                fi
                # If we are building a host-specific initramfs and this
                # module is not already loaded, move on to the next one.
                [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
                    && ! echo $add_drivers | grep -qe "\<${_mod}\>" \
                    && return

                # We use '-d' option in modprobe only if modules prefix path
                # differs from default '/'.  This allows us to use Dracut with
                # old version of modprobe which doesn't have '-d' option.
                local _moddirname=${KERNEL_MODS%%/lib/modules/*}
                [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"

                # ok, load the module, all its dependencies, and any firmware
                # it may require
                for_each_kmod_dep install_kmod_with_fw $_mod \
                    --set-version $KERNEL_VER ${_moddirname} $_mpargs
                ((_ret+=$?))
                ;;
        esac
        return $_ret
    }

    function instmods_1() {
        local _mod _mpargs
        if (($# == 0)); then  # filenames from stdin
            while read _mod; do
                inst1mod "${_mod%.ko*}" || {
                    if [ "$_check" = "yes" ]; then
                        dfatal "Failed to install $_mod"
                        return 1
                    fi
                }
            done
        fi
        while (($# > 0)); do  # filenames as arguments
            inst1mod ${1%.ko*} || {
                if [ "$_check" = "yes" ]; then
                    dfatal "Failed to install $1"
                    return 1
                fi
            }
            shift
        done
        return 0
    }

    local _ret _filter_not_found='FATAL: Module .* not found.'
    set -o pipefail
    # Capture all stderr from modprobe to _fderr. We could use {var}>...
    # redirections, but that would make dracut require bash4 at least.
    eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
    | while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
    _ret=$?
    set +o pipefail
    return $_ret
}

# inst_libdir_file [-n <pattern>] <file> [<file>...]
# Install a <file> located on a lib directory to the initramfs image
# -n <pattern> install non-matching files
inst_libdir_file() {
    if [[ "$1" == "-n" ]]; then
        local _pattern=$1
        shift 2
        for _dir in $libdirs; do
            for _i in "$@"; do
                for _f in "$_dir"/$_i; do
                    [[ "$_i" =~ $_pattern ]] || continue
                    [[ -e "$_i" ]] && dracut_install "$_i"
                done
            done
        done
    else
        for _dir in $libdirs; do
            for _i in "$@"; do
                for _f in "$_dir"/$_i; do
                    [[ -e "$_f" ]] && dracut_install "$_f"
                done
            done
        done
    fi
}

check_nspawn() {
    [[ -d /sys/fs/cgroup/systemd ]]
}


do_test() {
    if [[ $UID != "0" ]]; then
        echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
        exit 0
    fi

# Detect lib paths
    [[ $libdir ]] || for libdir in /lib64 /lib; do
        [[ -d $libdir ]] && libdirs+=" $libdir" && break
    done

    [[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
        [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
    done

    import_testdir
    import_initdir

    while (($# > 0)); do
        case $1 in
            --run)
                echo "TEST RUN: $TEST_DESCRIPTION"
                test_run
                ret=$?
                if [ $ret -eq 0 ]; then
                    echo "TEST RUN: $TEST_DESCRIPTION [OK]"
                else
                    echo "TEST RUN: $TEST_DESCRIPTION [FAILED]"
                fi
                exit $ret;;
            --setup)
                echo "TEST SETUP: $TEST_DESCRIPTION"
                test_setup
                exit $?;;
            --clean)
                echo "TEST CLEANUP: $TEST_DESCRIPTION"
                test_cleanup
                rm -fr "$TESTDIR"
                rm -f .testdir
                exit $?;;
            --all)
                echo -n "TEST: $TEST_DESCRIPTION ";
                (
                    test_setup && test_run
                    ret=$?
                    test_cleanup
                    rm -fr "$TESTDIR"
                    rm -f .testdir
                    exit $ret
                ) </dev/null >test.log 2>&1
                ret=$?
                if [ $ret -eq 0 ]; then
                    rm test.log
                    echo "[OK]"
                else
                    echo "[FAILED]"
                    echo "see $(pwd)/test.log"
                fi
                exit $ret;;
            *) break ;;
        esac
        shift
    done
}