Merge commit 'martins/master'

(This used to be ctdb commit 32a69b0efa078b069802470be6488a4efe32961d)

ctdb/tests/complex/44_failover_nfs_oneway.sh

@@ -66,10 +66,11 @@ ctdb_test_exit_hook_add rmdir "$mnt_d"
 
 echo "Create file containing random data..."
 dd if=/dev/urandom of=$local_f bs=1k count=1
+chmod 644 "$local_f" # needed for *_squash?
 local_sum=$(sum $local_f)
 [ $? -eq 0 ]
 
-scp "$local_f" "$remote_f"
+scp -p "$local_f" "$remote_f"
 
 echo "Mounting ${test_ip}:${first_export} on ${mnt_d} ..."
 mount -o timeo=1,hard,intr,vers=3 ${test_ip}:${first_export} ${mnt_d}

ctdb/tests/events.d/00.test

@@ -69,6 +69,11 @@ case $cmd in
 	echo "ctdb shutdown event"
 	exit 0
 	;;
+
+     stopped)
+	echo "ctdb stopped event"
+	exit 0
+	;;
 esac
 
 echo "Invalid command $cmd"

ctdb/tests/scripts/ctdb_test_functions.bash

@@ -326,7 +326,7 @@ _cluster_is_healthy ()
 	count=0
         while read line ; do
 	    count=$(($count + 1))
-	    [ "${line#:*:*:}" != "0:0:0:0:" ] && return 1
+	    [ "${line##:*:*:*1:}" != "$line" ] && return 1
         done
 	[ $count -gt 0 ] && return $?
     } <<<"$out" # Yay bash!
@@ -368,14 +368,16 @@ node_has_status ()
 
     local bits fpat mpat
     case "$status" in
-	(unhealthy)    bits="?:?:?:1" ;;
+	(unhealthy)    bits="?:?:?:1:*" ;;
-	(healthy)      bits="?:?:?:0" ;;
+	(healthy)      bits="?:?:?:0:*" ;;
-	(disconnected) bits="1:?:?:?" ;;
+	(disconnected) bits="1:*" ;;
-	(connected)    bits="0:?:?:?" ;;
+	(connected)    bits="0:*" ;;
-	(banned)       bits="?:1:?:?" ;;
+	(banned)       bits="?:1:*" ;;
-	(unbanned)     bits="?:0:?:?" ;;
+	(unbanned)     bits="?:0:*" ;;
-	(disabled)     bits="?:?:1:?" ;;
+	(disabled)     bits="?:?:1:*" ;;
-	(enabled)      bits="?:?:0:?" ;;
+	(enabled)      bits="?:?:0:*" ;;
+	(stopped)      bits="?:?:?:?:1:*" ;;
+	(notstopped)   bits="?:?:?:?:0:*" ;;
 	(frozen)       fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
 	(unfrozen)     fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
 	(monon)        mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
@@ -393,7 +395,7 @@ node_has_status ()
 	{
             read x
             while read line ; do
-		[ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0
+		[ "${line#:${pnn}:*:${bits}}" != "$line" ] && return 0
             done
 	    return 1
 	} <<<"$out" # Yay bash!

ctdb/tests/simple/21_ctdb_disablemonitor.sh

@@ -44,6 +44,9 @@ set -e
 
 cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 # We need this for later, so we know how long to sleep.

ctdb/tests/simple/41_ctdb_ban.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-test_info()
-{
-    cat <<EOF
-Verify the operation of the 'ctdb ban' command.
-
-This is a superficial test of the 'ctdb ban' command.  It trusts
-information from CTDB that indicates that the IP failover has
-happened correctly.  Another test should check that the failover
-has actually happened at the networking level.
-
-Prerequisites:
-
-* An active CTDB cluster with at least 2 active nodes.
-
-Steps:
-
-1. Verify that the status on all of the ctdb nodes is 'OK'.
-2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
-3. Before the ban timeout expires, verify that the status of the
-   node changes to 'banned'.
-4. Verify that the public IP addresses that were being served by
-   the node are failed over to one of the other nodes.
-5. When the ban expires ensure that the status of the node changes
-   back to 'OK' and that the public IP addresses move back to the
-   node.
-
-Expected results:
-
-* The status of the banned nodes changes as expected and IP addresses
-  failover as expected.
-EOF
-}
-
-. ctdb_test_functions.bash
-
-ctdb_test_init "$@"
-
-set -e
-
-cluster_is_healthy
-
-echo "Finding out which node is the recovery master..."
-try_command_on_node -v 0 "$CTDB recmaster"
-recmaster=$out
-
-echo "Getting list of public IPs..."
-try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
-
-# When selecting test_node we want a node that has public IPs and that
-# is not the recmaster.  We pick the first one that satisfies both
-# conditions.  We avoid the recmaster because banning the recmaster
-# (obviously) causes the recmaster to change... and changing the
-# recmaster causes all nodes to become unbanned!
-test_node=""
-
-ips=""
-while read ip pnn ; do
-    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
-    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
-done <<<"$out" # bashism to avoid problem setting variable in pipeline.
-
-if [ -z "$test_node" ] ; then
-    echo "BAD: unable to select a suitable node for banning."
-    exit 1
-fi
-
-echo "Selected node ${test_node} with IPs: $ips"
-
-ban_time=15
-
-echo "Banning node $test_node for $ban_time seconds"
-try_command_on_node 1 $CTDB ban $ban_time -n $test_node
-
-# Avoid a potential race condition...
-onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
-
-if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
-    echo "All IPs moved."
-else
-    echo "Some IPs didn't move."
-    testfailures=1
-fi
-
-echo "Sleeping until ban expires..."
-sleep_for $ban_time
-
-onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
-
-# BUG: this is only guaranteed if DeterministicIPs is 1 and
-#      NoIPFailback is 0.
-if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
-    echo "All IPs moved."
-else
-    echo "Some IPs didn't move."
-    testfailures=1
-fi

ctdb/tests/simple/41_ctdb_stop.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb stop' command.
+
+This is a superficial test of the 'ctdb stop' command.  It trusts
+information from CTDB that indicates that the IP failover has
+happened correctly.  Another test should check that the failover
+has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Stop one of the nodes using the 'ctdb stop' command.
+3. Verify that the status of the node changes to 'stopped'.
+4. Verify that the public IP addresses that were being served by
+   the node are failed over to one of the other nodes.
+
+Expected results:
+
+* The status of the stopped nodes changes as expected and IP addresses
+  failover as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
+
+select_test_node_and_ips
+
+echo "Stopping node ${test_node}..."
+try_command_on_node 1 $CTDB stop -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node stopped
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi

ctdb/tests/simple/42_ctdb_continue.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb continue' command.
+
+This is a superficial test of the 'ctdb continue' command.  It trusts
+information from CTDB that indicates that the IP failover and failback
+has happened correctly.  Another test should check that the failover
+and failback has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Stop one of the nodes using the 'ctdb stop' command.
+3. Verify that the status of the node changes to 'stopped'.
+4. Verify that the public IP addresses that were being served by
+   the node are failed over to one of the other nodes.
+5. Use 'ctdb continue' to bring the node back online.
+6. Verify that the status of the node changes back to 'OK' and that
+   the public IP addresses move back to the node.
+
+Expected results:
+
+* The 'ctdb continue' command successfully brings a stopped node online.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
+
+select_test_node_and_ips
+
+echo "Stopping node ${test_node}..."
+try_command_on_node 1 $CTDB stop -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node stopped
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "Continuing node $test_node"
+try_command_on_node 1 $CTDB continue -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node notstopped
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+#      NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi

ctdb/tests/simple/42_ctdb_unban.sh

@@ -1,95 +0,0 @@
-#!/bin/bash
-
-test_info()
-{
-    cat <<EOF
-Verify the operation of the 'ctdb unban' command.
-
-This is a superficial test of the 'ctdb uban' command.  It trusts
-information from CTDB that indicates that the IP failover and failback
-has happened correctly.  Another test should check that the failover
-and failback has actually happened at the networking level.
-
-Prerequisites:
-
-* An active CTDB cluster with at least 2 active nodes.
-
-Steps:
-
-1. Verify that the status on all of the ctdb nodes is 'OK'.
-2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
-3. Before the ban timeout expires, verify that the status of the
-   node changes to 'banned'.
-4. Verify that the public IP addresses that were being served by
-   the node are failed over to one of the other nodes.
-5. Before the ban timeout expires, use 'ctdb unban' to unban the
-   node.
-6. Verify that the status of the node changes back to 'OK' and that
-   the public IP addresses move back to the node.
-
-Expected results:
-
-* The 'ctdb unban' command successfully unbans a banned node.
-EOF
-}
-
-. ctdb_test_functions.bash
-
-ctdb_test_init "$@"
-
-set -e
-
-cluster_is_healthy
-
-echo "Finding out which node is the recovery master..."
-try_command_on_node -v 0 "$CTDB recmaster"
-recmaster=$out
-
-echo "Getting list of public IPs..."
-try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
-
-# See 41_ctdb_ban.sh for an explanation of why test_node is chosen
-# like this.
-test_node=""
-
-ips=""
-while read ip pnn ; do
-    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
-    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
-done <<<"$out" # bashism to avoid problem setting variable in pipeline.
-
-if [ -z "$test_node" ] ; then
-    echo "BAD: unable to select a suitable node for banning."
-    exit 1
-fi
-
-echo "Selected node ${test_node} with IPs: $ips"
-
-ban_time=60
-
-echo "Banning node $test_node for $ban_time seconds"
-try_command_on_node 1 $CTDB ban $ban_time -n $test_node
-
-# Avoid a potential race condition...
-onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
-
-if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
-    echo "All IPs moved."
-else
-    echo "Some IPs didn't move."
-    testfailures=1
-fi
-
-echo "Unbanning node $test_node"
-try_command_on_node 1 $CTDB unban -n $test_node
-
-onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
-
-# BUG: this is only guaranteed if DeterministicIPs is 1 and
-#      NoIPFailback is 0.
-if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
-    echo "All IPs moved."
-else
-    echo "Some IPs didn't move."
-    testfailures=1
-fi

ctdb/tests/simple/43_stop_recmaster_yield.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb stop' causes a node to yield the recovery master role.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Determine which node is the recmaster.
+2. Stop this node using the 'ctdb stop' command.
+3. Verify that the status of the node changes to 'stopped'.
+4. Verify that this node no longer has the recovery master role.
+
+Expected results:
+
+* The 'ctdb stop' command causes a node to yield the recmaster role.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
+
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+test_node=$out
+
+echo "Stopping node ${test_node} - it is the current recmaster..."
+try_command_on_node 1 $CTDB stop -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node stopped
+
+echo "Checking which node is the recovery master now..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
+if [ "$recmaster" != "$test_node" ] ; then
+    echo "OK: recmaster moved to node $recmaster"
+else
+    echo "BAD: recmaster did not move"
+    exit 1
+fi

ctdb/tools/onnode

@@ -131,10 +131,10 @@ get_nodes_with_status ()
     local bits
     case "$status" in
 	healthy)
-	    bits="0:0:0:0"
+	    bits="0:0:0:0:0"
 	    ;;
 	connected)
-	    bits="0:[0-1]:[0-1]:[0-1]"
+	    bits="0:[0-1]:[0-1]:[0-1]:[0-1]"
 	    ;;
 	*)
 	    invalid_nodespec