test: add a basic multipath test + failover

test/TEST-64-UDEV-STORAGE/test.sh

@@ -23,11 +23,16 @@ test_append_files() {
         instmods "=block" "=md" "=nvme" "=scsi"
         install_dmevent
         generate_module_dependencies
-        inst_binary lsblk
-        inst_binary wc
+        image_install lsblk wc
+
+        # Configure multipath
+        if command -v multipath && command -v multipathd; then
+            install_multipath
+        fi
 
         for i in {0..127}; do
             dd if=/dev/zero of="${TESTDIR:?}/disk$i.img" bs=1M count=1
+            echo "device$i" >"${TESTDIR:?}/disk$i.img"
         done
     )
 }
@@ -182,6 +187,49 @@ EOF
     QEMU_SMP=1 QEMU_TIMEOUT=60 test_run_one "${1:?}"
 }
 
+testcase_multipath_basic_failover() {
+    if ! command -v multipath || ! command -v multipathd; then
+        echo "Missing multipath tools, skipping the test..."
+        return 77
+    fi
+
+    local qemu_opts=("-device virtio-scsi-pci,id=scsi")
+    local partdisk="${TESTDIR:?}/multipathpartitioned.img"
+    local image lodev nback ndisk wwn
+
+    if [[ ! -e "$partdisk" ]]; then
+        dd if=/dev/zero of="$partdisk" bs=1M count=16
+        lodev="$(losetup --show -f -P "$partdisk")"
+        sfdisk "${lodev:?}" <<EOF
+label: gpt
+
+name="first_partition", size=5M
+uuid="deadbeef-dead-dead-beef-000000000000", name="failover_part", size=5M
+EOF
+        udevadm settle
+        mkfs.ext4 -U "deadbeef-dead-dead-beef-111111111111" -L "failover_vol" "${lodev}p2"
+        losetup -d "$lodev"
+    fi
+
+    # Add 64 multipath devices, each backed by 4 paths
+    for ndisk in {0..63}; do
+        wwn="0xDEADDEADBEEF$(printf "%.4d" "$ndisk")"
+        # Use a partitioned disk for the first device to test failover
+        [[ $ndisk -eq 0 ]] && image="$partdisk" || image="${TESTDIR:?}/disk$ndisk.img"
+
+        for nback in {0..3}; do
+            qemu_opts+=(
+                "-device scsi-hd,drive=drive${ndisk}x${nback},serial=MPIO$ndisk,wwn=$wwn"
+                "-drive format=raw,cache=unsafe,file=$image,file.locking=off,if=none,id=drive${ndisk}x${nback}"
+            )
+        done
+    done
+
+    KERNEL_APPEND="systemd.setenv=TEST_FUNCTION_NAME=${FUNCNAME[0]} ${USER_KERNEL_APPEND:-}"
+    QEMU_OPTIONS="${qemu_opts[*]} ${USER_QEMU_OPTIONS:-}"
+    test_run_one "${1:?}"
+}
+
 # Allow overriding which tests should be run from the "outside", useful for manual
 # testing (make -C test/... TESTCASES="testcase1 testcase2")
 if [[ -v "TESTCASES" && -n "$TESTCASES" ]]; then

test/units/testsuite-64.sh

@@ -19,6 +19,105 @@ testcase_virtio_scsi_identically_named_partitions() {
     [[ "$(lsblk --noheadings -a -o NAME,PARTLABEL | grep -c "Hello world")" -eq $((16 * 8)) ]]
 }
 
+testcase_multipath_basic_failover() {
+    local dmpath i path wwid
+
+    # Configure multipath
+    cat >/etc/multipath.conf <<\EOF
+defaults {
+    # Use /dev/mapper/$WWN paths instead of /dev/mapper/mpathX
+    user_friendly_names no
+    find_multipaths yes
+    enable_foreign "^$"
+}
+
+blacklist_exceptions {
+    property "(SCSI_IDENT_|ID_WWN)"
+}
+
+blacklist {
+}
+EOF
+    modprobe -v dm_multipath
+    systemctl start multipathd.service
+    systemctl status multipathd.service
+    multipath -ll
+    ls -l /dev/disk/by-id/
+
+    for i in {0..63}; do
+        wwid="deaddeadbeef$(printf "%.4d" "$i")"
+        path="/dev/disk/by-id/wwn-0x$wwid"
+        dmpath="$(readlink -f "$path")"
+
+        lsblk "$path"
+        multipath -C "$dmpath"
+        # We should have 4 active paths for each multipath device
+        [[ "$(multipath -l "$path" | grep -c running)" -eq 4 ]]
+    done
+
+    # Test failover (with the first multipath device that has a partitioned disk)
+    echo "${FUNCNAME[0]}: test failover"
+    local device expected link mpoint part
+    local -a devices
+    mpoint="$(mktemp -d /mnt/mpathXXX)"
+    wwid="deaddeadbeef0000"
+    path="/dev/disk/by-id/wwn-0x$wwid"
+
+    # All following symlinks should exists and should be valid
+    local -a part_links=(
+        "/dev/disk/by-id/wwn-0x$wwid-part2"
+        "/dev/disk/by-partlabel/failover_part"
+        "/dev/disk/by-partuuid/deadbeef-dead-dead-beef-000000000000"
+        "/dev/disk/by-label/failover_vol"
+        "/dev/disk/by-uuid/deadbeef-dead-dead-beef-111111111111"
+    )
+    for link in "${part_links[@]}"; do
+        test -e "$link"
+    done
+
+    # Choose a random symlink to the failover data partition each time, for
+    # a better coverage
+    part="${part_links[$RANDOM % ${#part_links[@]}]}"
+
+    # Get all devices attached to a specific multipath device (in H:C:T:L format)
+    # and sort them in a random order, so we cut off different paths each time
+    mapfile -t devices < <(multipath -l "$path" | grep -Eo '[0-9]+:[0-9]+:[0-9]+:[0-9]+' | sort -R)
+    if [[ "${#devices[@]}" -ne 4 ]]; then
+        echo "Expected 4 devices attached to WWID=$wwid, got ${#devices[@]} instead"
+        return 1
+    fi
+    # Drop the last path from the array, since we want to leave at least one path active
+    unset "devices[3]"
+    # Mount the first multipath partition, write some data we can check later,
+    # and then disconnect the remaining paths one by one while checking if we
+    # can still read/write from the mount
+    mount -t ext4 "$part" "$mpoint"
+    expected=0
+    echo -n "$expected" >"$mpoint/test"
+    # Sanity check we actually wrote what we wanted
+    [[ "$(<"$mpoint/test")" == "$expected" ]]
+
+    for device in "${devices[@]}"; do
+        echo offline >"/sys/class/scsi_device/$device/device/state"
+        [[ "$(<"$mpoint/test")" == "$expected" ]]
+        expected="$((expected + 1))"
+        echo -n "$expected" >"$mpoint/test"
+
+        # Make sure all symlinks are still valid
+        for link in "${part_links[@]}"; do
+            test -e "$link"
+        done
+    done
+
+    multipath -l "$path"
+    # Three paths should be now marked as 'offline' and one as 'running'
+    [[ "$(multipath -l "$path" | grep -c offline)" -eq 3 ]]
+    [[ "$(multipath -l "$path" | grep -c running)" -eq 1 ]]
+
+    umount "$mpoint"
+    rm -fr "$mpoint"
+}
+
 : >/failed
 
 udevadm settle