sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2024-12-24 21:34:56 +03:00
Code

ctdb-tests: Clean up some tests where IP movement is checked

Browse Source 
Some of this implements logic that exists in functions.  Some of it is
overly complicated and potentially failure-prone.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
This commit is contained in:
Martin Schwenke 2014-09-12 13:20:43 +10:00 committed by Amitay Isaacs
parent 81a8758b9b
commit 371543207e
9 changed files with 85 additions and 411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
View File

@ -5,36 +5,7 @@ test_info()
cat <<EOF
Verify that a node's public IP address can be deleted using 'ctdb deleteip'.
Check that the address is actually deleted from the interface.
Prerequisites:
* An active CTDB cluster with at least 2 active nodes.
* Test must be run on a real or virtual cluster rather than against
local daemons. There is nothing intrinsic to this test that forces
this - it is because tests run against local daemons don't use the
regular eventscripts. Local daemons put public addresses on
loopback, so we can't reliably test when IPs have moved between
nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
served.
3. Select an IP address being served by the node and check that it
actually appears on the interface it is supposed to be on.
4. Delete the IP address using 'ctdb delip'.
5. Verify that the deleted IP address is no longer listed using the
all_ips_on_node helper function.
6. Verify that the deleted IP address no longer appears on the
interface it was on.
Expected results:
* 'ctdb delip' removes an IP address from the list of public IP
addresses being served by a node and from the network interface.
This is an extended version of simple/17_ctdb_config_delete_ip.sh
EOF
}
@ -51,55 +22,23 @@ cluster_is_healthy
# Reset configuration
ctdb_restart_when_done
echo "Getting list of public IPs..."
all_ips_on_node -v 0
select_test_node_and_ips
get_test_ip_mask_and_iface
# Select an IP/node to remove.
num_ips=$(echo "$out" | wc -l)
num_to_remove=$(($RANDOM % $num_ips))
echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..."
try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${test_ip}/'"
# Find the details in the list.
i=0
while [ $i -le $num_to_remove ] ; do
read ip_to_remove test_node
i=$(($i + 1))
done <<<"$out"
echo "Determining interface for ${ip_to_remove} on ${test_node}."
try_command_on_node $test_node "ctdb ip -Y -v"
iface=$(echo "$out" | awk -F: -v ip=${ip_to_remove} -v pnn=${test_node} '$2 == ip && $3 == pnn { print $4 }')
echo "$iface"
[ -n "$iface" ]
echo "Checking that node ${test_node} hosts ${ip_to_remove} on interface ${iface}..."
try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${ip_to_remove}/'"
echo "Attempting to remove ${ip_to_remove} from node ${test_node}."
try_command_on_node $test_node $CTDB delip $ip_to_remove
echo "Sleeping..."
sleep_for 1
test_node_ips=""
while read ip pnn ; do
[ "$pnn" = "$test_node" ] && \
test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
done <<<"$out" # bashism to avoid problem setting variable in pipeline.
if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
echo "GOOD: That worked!"
else
echo "BAD: The remove IP address is still there!"
testfailures=1
fi
echo "Attempting to remove ${test_ip} from node ${test_node}."
try_command_on_node $test_node $CTDB delip $test_ip
wait_until_ips_are_on_node '!' $test_node $test_ip
timeout=60
increment=5
count=0
echo "Waiting for ${ip_to_remove} to disappear from ${iface}..."
echo "Waiting for ${test_ip} to disappear from ${iface}..."
while : ; do
try_command_on_node -v $test_node "ip addr show dev $iface"
if echo "$out" | grep -E 'inet[[:space:]]*${ip_to_remove}/'; then
if echo "$out" | grep -E 'inet[[:space:]]*${test_ip}/'; then
echo "Still there..."
if [ $(($count * $increment)) -ge $timeout ] ; then
echo "BAD: Timed out waiting..."

5
ctdb/tests/scripts/integration.bash
View File

@ -152,7 +152,8 @@ sanity_check_ips ()
prev="$ipp"
done <<<"$ips"
echo "BAD: a node was -1 or IPs are only assigned to one node"
echo "BAD: a node was -1 or IPs are only assigned to one node:"
echo "$ips"
echo "Are you running an old version of CTDB?"
return 1
}
@ -483,7 +484,7 @@ node_has_some_ips ()
wait_until_node_has_some_ips ()
{
echo "Waiting for node to have some IPs..."
echo "Waiting for some IPs to be assigned to node ${test_node}"
wait_until 60 node_has_some_ips "$@"
}

111
ctdb/tests/simple/16_ctdb_config_add_ip.sh
View File

@ -5,32 +5,8 @@ test_info()
cat <<EOF
Verify that an IP address can be added to a node using 'ctdb addip'.
This test goes to some trouble to figure out which IP address to add
but assumes a 24-bit subnet mask. It does not handle IPv6. It does
not do any network level checks that the new IP address is reachable
but simply trusts 'ctdb ip' that the address has been added. There is
also an extra prerequisite that the node being added to already has
public addresses - this is difficult to avoid if the extra address is
to be sensibly chosen.
Prerequisites:
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
served.
3. Add an additional public address to be served by the node, using
'ctdb addip'.
4. Verify that this IP address has been added to the list of IP
addresses being served by the node, using the 'ctdb ip' command.
Expected results:
* 'ctdb ip' adds an IP address to the list of public IP addresses
being served by a node.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -45,79 +21,16 @@ cluster_is_healthy
# Reset configuration
ctdb_restart_when_done
echo "Getting list of public IPs..."
all_ips_on_node 0
select_test_node_and_ips
get_test_ip_mask_and_iface
# When selecting test_node we just want a node that has public IPs.
# This will work and is economically semi-randomly. :-)
read x test_node <<<"$out"
echo "Deleting IP $test_ip from all nodes"
try_command_on_node $test_node $CTDB delip -n all $test_ip
wait_until_ips_are_on_node '!' $test_node $test_ip
test_node_ips=""
all_ips=""
while read ip pnn ; do
all_ips="${all_ips}${all_ips:+ }${ip}"
[ "$pnn" = "$test_node" ] && \
test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
done <<<"$out"
# Debugging...
try_command_on_node -v all $CTDB ip
echo "Selected node ${test_node} with IPs: $test_node_ips"
# Try to find a free IP adddress. This is inefficient but should
# succeed quickly.
if [ -z "$TEST_LOCAL_DAEMONS" ] ; then
try_command_on_node $test_node "ip addr show"
all_test_node_ips=$(echo "$out" | sed -rn -e 's@^[[:space:]]+inet[[:space:]]+([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+/[[:digit:]]+).*[[:space:]]([^[:space:]]+)+$@\1:\2@p')
else
all_test_node_ips=""
fi
add_ip=""
# Use an IP already on one of the nodes, remove the last octet and
# loop through the possible IP addreses.
for i in $test_node_ips ; do
prefix="${i%.*}"
for j in $(seq 101 199) ; do
try="${prefix}.${j}"
# Try to make sure it isn't used anywhere!
# First, make sure it isn't an existing public address on the
# cluster.
for k in $all_ips ; do
[ "$try" = "$k" ] && continue 2
done
# Also make sure it isn't some other address in use on the
# node.
for k in $all_test_node_ips ; do
[ "$try" = "${k%/*}" ] && continue 2
done
# Get the interface details for $i, which our address is a
# close relative of. This should never fail but it can't hurt
# to be careful...
try_command_on_node $test_node "ctdb ip -v -Y"
while IFS=":" read x ip pnn iface x ; do
if [ "$i" = "$ip" ]; then
add_ip="$try/32:$iface"
break 3
fi
done <<<"$out"
done
done
if [ -z "$add_ip" ] ; then
echo "BAD: Unable to find IP address to add."
exit 1
fi
echo "Adding IP: ${add_ip/:/ on interface }"
try_command_on_node $test_node $CTDB addip ${add_ip/:/ }
echo "Waiting for IP to be added..."
if wait_until 60 ips_are_on_node $test_node ${add_ip%/*} ; then
echo "That worked!"
else
echo "BAD: IP didn't get added."
try_command_on_node $test_node $CTDB ip -n all
exit 1
fi
echo "Adding IP ${test_ip}/${mask} on ${iface}, node ${test_node}"
try_command_on_node $test_node $CTDB addip ${test_ip}/${mask} $iface
wait_until_ips_are_on_node $test_node $test_ip

59
ctdb/tests/simple/17_ctdb_config_delete_ip.sh
View File

@ -5,28 +5,8 @@ test_info()
cat <<EOF
Verify that a node's public IP address can be deleted using 'ctdb deleteip'.
This test does not do any network level checks that the IP address is
no longer reachable but simply trusts 'ctdb ip' that the address has
been deleted.
Prerequisites:
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
served.
3. Delete one public IP address being be served by the node, using
'ctdb delip'.
4. Verify that the delete IP address is no longer listed using the
all_ips_on_node helper function.
Expected results:
* 'ctdb delip' removes an IP address from the list of public IP
addresses being served by a node.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -41,35 +21,8 @@ cluster_is_healthy
# Reset configuration
ctdb_restart_when_done
echo "Getting list of public IPs..."
all_ips_on_node -v 0
select_test_node_and_ips
# Select an IP/node to remove.
num_ips=$(echo "$out" | wc -l)
num_to_remove=$(($RANDOM % $num_ips))
# Find the details in the list.
i=0
while [ $i -le $num_to_remove ] ; do
read ip_to_remove test_node
i=$(($i + 1))
done <<<"$out"
echo "Attempting to remove ${ip_to_remove} from node ${test_node}."
try_command_on_node $test_node $CTDB delip $ip_to_remove
echo "Sleeping..."
sleep_for 1
test_node_ips=""
while read ip pnn ; do
[ "$pnn" = "$test_node" ] && \
test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
done <<<"$out" # bashism to avoid problem setting variable in pipeline.
if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
echo "GOOD: That worked!"
else
echo "BAD: The remove IP address is still there!"
testfailures=1
fi
echo "Deleting IP ${test_ip} from node ${test_node}"
try_command_on_node $test_node $CTDB delip $test_ip
wait_until_ips_are_on_node '!' $test_node $test_ip

94
ctdb/tests/simple/23_ctdb_moveip.sh
View File

@ -5,27 +5,10 @@ test_info()
cat <<EOF
Verify that 'ctdb moveip' allows movement of public IPs between cluster nodes.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
To work, this test unsets DeterministicIPs and sets NoIPFailback.
This test does not do any network level checks that the IP address is
no longer reachable but simply trusts 'ctdb ip' that the address has
been deleted.
Prerequisites:
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
served.
3. Use 'ctdb moveip' to move an address from one node to another.
4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node.
Expected results:
* 'ctdb moveip' allows an IP address to be moved between cluster nodes.
EOF
}
@ -40,62 +23,39 @@ cluster_is_healthy
# Reset configuration
ctdb_restart_when_done
try_command_on_node 0 "$CTDB listnodes | wc -l"
num_nodes="$out"
echo "There are $num_nodes nodes..."
if [ $num_nodes -lt 2 ] ; then
echo "Less than 2 nodes!"
exit 1
fi
echo "Getting list of public IPs..."
all_ips_on_node -v 0
select_test_node_and_ips
sanity_check_ips "$out"
# Select an IP/node to move.
num_ips=$(echo "$out" | wc -l)
num_to_move=$(($RANDOM % $num_ips))
# Find the details in the list.
i=0
while [ $i -le $num_to_move ] ; do
read ip_to_move test_node
i=$(($i + 1))
done <<<"$out"
# Can only move address to a node that is willing to host $ip_to_move.
# This inefficient but shouldn't take long or get stuck.
to_node=$test_node
while [ $test_node -eq $to_node ] ; do
n=$(($RANDOM % $num_ips))
i=0
while [ $i -le $n ] ; do
read x to_node
i=$(($i + 1))
# Find a target node - it must be willing to host $test_ip
try_command_on_node any "$CTDB listnodes | wc -l"
num_nodes="$out"
to_node=""
for i in $(seq 0 $(($num_nodes - 1)) ) ; do
[ $i -ne $test_node ] || continue
all_ips_on_node $i
while read ip x ; do
if [ "$ip" = "$test_ip" ] ; then
to_node="$i"
break 2
fi
done <<<"$out"
done
if [ -z "$to_node" ] ; then
echo "Unable to find target node"
exit 1
fi
echo "Target node is ${to_node}"
echo "Turning off DeterministicIPs..."
try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all
echo "Turning on NoIPFailback..."
try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all
echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}."
try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node
if wait_until_ips_are_on_node '!' $test_node $ip_to_move ; then
echo "IP moved from ${test_node}."
else
echo "BAD: IP didn't move from ${test_node}."
exit 1
fi
if wait_until_ips_are_on_node "$to_node" $ip_to_move ; then
echo "IP moved to ${to_node}."
else
echo "BAD: IP didn't move to ${to_node}."
exit 1
fi
echo "Attempting to move ${test_ip} from node ${test_node} to node ${to_node}"
try_command_on_node $test_node $CTDB moveip $test_ip $to_node
wait_until_ips_are_on_node '!' $test_node $test_ip
wait_until_ips_are_on_node $to_node $test_ip

35
ctdb/tests/simple/31_ctdb_disable.sh
View File

@ -5,27 +5,13 @@ test_info()
cat <<EOF
Verify the operation of 'ctdb disable'.
This is a superficial test of the 'ctdb disable' command. It trusts
information from CTDB that indicates that the IP failover has happened
correctly. Another test should check that the failover has actually
happened at the networking level.
* Verify that the status of the node changes to 'disabled'.
Prerequisites:
* Verify that the IP addreses served by the disabled node are failed
over to other nodes.
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Disable one of the nodes using 'ctdb disable -n <node>'.
3. Verify that the status of the node changes to 'disabled'.
4. Verify that the IP addreses served by the disabled node are failed
over to other nodes.
Expected results:
* The status of the disabled node changes as expected and IP addresses
failover as expected.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -43,15 +29,6 @@ ctdb_restart_when_done
select_test_node_and_ips
echo "Disabling node $test_node"
try_command_on_node 1 $CTDB disable -n $test_node
# Avoid a potential race condition...
wait_until_node_has_status $test_node disabled
if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then
echo "All IPs moved."
else
echo "Some IPs didn't move."
testfailures=1
fi
wait_until_ips_are_on_node '!' $test_node $test_node_ips

41
ctdb/tests/simple/32_ctdb_enable.sh
View File

@ -5,31 +5,15 @@ test_info()
cat <<EOF
Verify the operation of 'ctdb enable'.
This is a superficial test of the 'ctdb enable' command. It trusts
information from CTDB that indicates that the IP failover has happened
correctly. Another test should check that the failover has actually
happened at the networking level.
After disabling a node...
Prerequisites:
* Verify that the status of a re-enabled node changes back to 'OK'.
* An active CTDB cluster with at least 2 active nodes.
* Verify that some public IP addreses are rebalanced to a re-enabled
node.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Disable one of the nodes using 'ctdb disable -n <node>'.
3. Verify that the status of the node changes to 'disabled'.
4. Verify that the public IP addreses served by the disabled node are
failed over to other nodes.
5. Enable the disabled node using 'ctdb enable -n '<node>'.
6. Verify that the status changes back to 'OK'.
7. Verify that some public IP addreses are failed back to the node.
Expected results:
* The status of a re-enabled node changes as expected and IP addresses
fail back as expected.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -47,19 +31,12 @@ select_test_node_and_ips
echo "Disabling node $test_node"
try_command_on_node 1 $CTDB disable -n $test_node
wait_until_node_has_status $test_node disabled
if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then
echo "All IPs moved."
else
echo "Some IPs didn't move."
testfailures=1
fi
echo "Waiting for IPs to no longer be hosted on node ${test_node}"
wait_until_ips_are_on_node '!' $test_node $test_node_ips
echo "Reenabling node $test_node"
echo "Re-enabling node $test_node"
try_command_on_node 1 $CTDB enable -n $test_node
wait_until_node_has_status $test_node enabled
wait_until_node_has_some_ips "$test_node"

33
ctdb/tests/simple/41_ctdb_stop.sh
View File

@ -5,27 +5,13 @@ test_info()
cat <<EOF
Verify the operation of the 'ctdb stop' command.
This is a superficial test of the 'ctdb stop' command. It trusts
information from CTDB that indicates that the IP failover has
happened correctly. Another test should check that the failover
has actually happened at the networking level.
* Verify that the status of the node changes to 'stopped'.
Prerequisites:
* Verify that the public IP addresses that were being served by
the node are failed over to one of the other nodes.
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Stop one of the nodes using the 'ctdb stop' command.
3. Verify that the status of the node changes to 'stopped'.
4. Verify that the public IP addresses that were being served by
the node are failed over to one of the other nodes.
Expected results:
* The status of the stopped nodes changes as expected and IP addresses
failover as expected.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -44,12 +30,5 @@ select_test_node_and_ips
echo "Stopping node ${test_node}..."
try_command_on_node 1 $CTDB stop -n $test_node
wait_until_node_has_status $test_node stopped
if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then
echo "All IPs moved."
else
echo "Some IPs didn't move."
testfailures=1
fi
wait_until_ips_are_on_node '!' $test_node $test_node_ips

37
ctdb/tests/simple/42_ctdb_continue.sh
View File

@ -5,29 +5,13 @@ test_info()
cat <<EOF
Verify the operation of the 'ctdb continue' command.
This is a superficial test of the 'ctdb continue' command. It trusts
information from CTDB that indicates that the IP failover and failback
has happened correctly. Another test should check that the failover
and failback has actually happened at the networking level.
After stopping a node...
Prerequisites:
* Verify that the status of the node changes back to 'OK' and that
some public IP addresses move back to the node.
* An active CTDB cluster with at least 2 active nodes.
Steps:
1. Verify that the status on all of the ctdb nodes is 'OK'.
2. Stop one of the nodes using the 'ctdb stop' command.
3. Verify that the status of the node changes to 'stopped'.
4. Verify that the public IP addresses that were being served by
the node are failed over to one of the other nodes.
5. Use 'ctdb continue' to bring the node back online.
6. Verify that the status of the node changes back to 'OK' and that
some public IP addresses move back to the node.
Expected results:
* The 'ctdb continue' command successfully brings a stopped node online.
This test does not do any network level checks to make sure IP
addresses are actually on interfaces. It just consults "ctdb ip".
EOF
}
@ -46,19 +30,10 @@ select_test_node_and_ips
echo "Stopping node ${test_node}..."
try_command_on_node 1 $CTDB stop -n $test_node
wait_until_node_has_status $test_node stopped
if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then
echo "All IPs moved."
else
echo "Some IPs didn't move."
testfailures=1
fi
wait_until_ips_are_on_node '!' $test_node $test_node_ips
echo "Continuing node $test_node"
try_command_on_node 1 $CTDB continue -n $test_node
wait_until_node_has_status $test_node notstopped
wait_until_node_has_some_ips "$test_node"