Separate test cleanup code in output and clean up ctdb restart code.

* ctdb_restart_when_done() now schedules a restart by setting an
  explicit variable that is respected in ctdb_test_exit(), rather than
  adding a restart to $ctdb_test_exit_hook.  This means that restarts
  are all done in one place.

* ctdb_test_exit() turns off "set -e" to make sure that all cleanup
  happens.

* ctdb_test_exit() now prints a clear message indicating where the
  test ends and the cleanup begins.  This message also includes the
  return code of the test.

* Add debug in cluster_is_healthy to try to capture information about
  unexpected unhealthiness when a test starts.

* Simplify simple/07_ctdb_process_exists.sh so that the exit code is
  generated more obviously.

* Remove redundant calls to ctdb_test_exit at the end of tests, since
  they're done automatically via a trap.  Also remove any preceding
  warnings of restarts or final hints about test success/failure.

* Allow multi-digit debug levels in simple/12_ctdb_getdebug.sh and
  simple/13_ctdb_setdebug.sh.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 56ece515e047a54f33e8b07726e52ba21a1d67e1)

ctdb/tests/scripts/ctdb_test_functions.bash

@@ -60,12 +60,20 @@ ctdb_test_exit ()
     trap - 0
 
     [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
+    status=$(($testfailures+0))
+
+    # Avoid making a test fail from this point onwards.  The test is
+    # now complete.
+    set +e
+
+    echo "*** TEST COMPLETE (RC=$status), CLEANING UP..."
 
     eval "$ctdb_test_exit_hook" || true
     unset ctdb_test_exit_hook
 
-    if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
-	echo "Restarting ctdb on all nodes to get back into known state..."
+    if $ctdb_test_restart_scheduled || \
+	! onnode 0 CTDB_TEST_CLEANING_UP=1 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
+
 	restart_ctdb
     else
 	# This could be made unconditional but then we might get
@@ -76,7 +84,7 @@ ctdb_test_exit ()
 	onnode 0 ctdb recover
     fi
 
-    test_exit
+    exit $status
 }
 
 ctdb_test_exit_hook_add ()
@@ -149,6 +157,7 @@ ctdb_test_init ()
 {
     scriptname=$(basename "$0")
     testfailures=0
+    ctdb_test_restart_scheduled=false
 
     ctdb_test_cmd_options $@
 
@@ -309,6 +318,14 @@ cluster_is_healthy ()
 	exit 0
     else
 	echo "Cluster is UNHEALTHY"
+	if [ -z "$CTDB_TEST_CLEANING_UP" ] ; then
+	    echo "DEBUG:"
+	    local i
+	    for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do
+		echo "$i"
+		$i || true
+	    done
+	fi
 	exit 1
     fi
 }
@@ -591,9 +608,11 @@ setup_ctdb ()
 
 restart_ctdb ()
 {
-    if [ "$1" = "-v" ] ; then
-	echo "Restarting CTDB (scheduled)..."
+    echo -n "Restarting CTDB"
+    if $ctdb_test_restart_scheduled ; then
+	echo -n " (scheduled)"
     fi
+    echo "..."
     
     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 	daemons_stop
@@ -621,11 +640,9 @@ restart_ctdb ()
 
 ctdb_restart_when_done ()
 {
-    ctdb_test_exit_hook_add restart_ctdb -v
+    ctdb_test_restart_scheduled=true
 }
 
-
-
 #######################################
 
 install_eventscript ()

ctdb/tests/simple/00_ctdb_init.sh

@@ -28,6 +28,5 @@ ctdb_test_init "$@"
 
 set -e
 
-echo "Restarting ctdb on all nodes..."
 setup_ctdb
 restart_ctdb

ctdb/tests/simple/01_ctdb_version.sh

@@ -48,5 +48,3 @@ else
     echo "BAD: CTDB version != RPM version"
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/02_ctdb_listvars.sh

@@ -39,5 +39,3 @@ sanity_check_output \
     5 \
     '^[[:alpha:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
     "$out"
-
-ctdb_test_exit

ctdb/tests/simple/03_ctdb_getvar.sh

@@ -49,7 +49,3 @@ while read var x val ; do
 	exit 1
     fi
 done
-
-testfailures=$?
-
-ctdb_test_exit

ctdb/tests/simple/04_ctdb_setvar.sh

@@ -77,7 +77,3 @@ fi
 echo "Look's good!  Putting the old value back..."
 cmd="$CTDB setvar $var $val"
 try_command_on_node 0 $cmd
-
-echo "All done..."
-
-ctdb_test_exit

ctdb/tests/simple/05_ctdb_listnodes.sh

@@ -57,5 +57,3 @@ while [ $n -lt $num_nodes ] ; do
     fi
     n=$(($n + 1))
 done
-
-ctdb_test_exit

ctdb/tests/simple/06_ctdb_getpid.sh

@@ -82,5 +82,3 @@ while [ $n -lt $num_nodes ] ; do
     fi
     n=$(($n + 1))
 done <<<"$pids_onnode"
-
-ctdb_test_exit

ctdb/tests/simple/07_ctdb_process_exists.sh

@@ -63,16 +63,4 @@ try_command_on_node $test_node 'echo $$'
 pid="$out"
 
 echo "Checking for PID $pid on node $test_node"
-# set -e is good, but avoid it here
-status=0
-onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
-echo "$out"
-
-if [ $status -ne 0 ] ; then
-    echo "OK"
-else
-    echo "BAD"
-    testfailures=1
-fi
-
-ctdb_test_exit
+try_command_on_node -v 0 "! $CTDB process-exists ${test_node}:${pid}"

ctdb/tests/simple/08_ctdb_isnotrecmaster.sh

@@ -54,5 +54,3 @@ else
     echo "BAD, there are only ${num_not_rm_lines} nodes claiming not to be the recmaster"
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/09_ctdb_ping.sh

@@ -52,7 +52,3 @@ sanity_check_output \
     1 \
     "(: ctdb_control error: 'ctdb_control to disconnected node'|Unable to get ping response from node 1|Node 1 is DISCONNECTED)" \
     "$out"
-
-echo "Expect a restart..."
-
-ctdb_test_exit

ctdb/tests/simple/11_ctdb_ip.sh

@@ -59,5 +59,3 @@ else
     echo "$out"
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/12_ctdb_getdebug.sh

@@ -38,7 +38,7 @@ getdebug_onnode="$out"
 
 sanity_check_output \
     $num_nodes \
-    '^Node [[:digit:]]+ is at debug level [[:alpha:]]+ \([[:digit:]]\)$' \
+    '^Node [[:digit:]]+ is at debug level [[:alpha:]]+ \([[:digit:]]+\)$' \
     "$out"
 
 try_command_on_node -v 1 "$CTDB getdebug -n all"
@@ -65,7 +65,7 @@ colons=""
 nl="
 "
 while read line ; do
-    t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]])\)$@:\1:\2:@')
+    t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]]+)\)$@:\1:\2:@')
     colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}"
 done <<<"$getdebug_onnode"
 
@@ -80,5 +80,3 @@ else
     echo "$out"
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/13_ctdb_setdebug.sh

@@ -47,7 +47,7 @@ get_debug ()
     
     try_command_on_node -v $node "$CTDB getdebug"
     check_debug=$(echo "$out" |
-	sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \(-?[[:digit:]]\)$@\1@')
+	sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \(-?[[:digit:]]+\)$@\1@')
 }
 
 set_and_check_debug ()
@@ -81,5 +81,3 @@ if [ "$testfailures" != 1 ] ; then
     echo "Returning the debug level to its initial value..."
     set_and_check_debug 1 "$initial_debug"
 fi
-
-ctdb_test_exit

ctdb/tests/simple/14_ctdb_statistics.sh

@@ -42,5 +42,3 @@ sanity_check_output 40 "$pattern" "$out"
 try_command_on_node -v 1 "$CTDB statistics -n all"
 
 sanity_check_output 40 "$pattern" "$out"
-
-ctdb_test_exit

ctdb/tests/simple/15_ctdb_statisticsreset.sh

@@ -81,5 +81,3 @@ while [ $n -lt $num_nodes ] ; do
 
     n=$(($n + 1))
 done
-
-ctdb_test_exit

ctdb/tests/simple/16_ctdb_config_add_ip.sh

@@ -114,5 +114,3 @@ else
     echo "BAD: Unable to find IP address to add."
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/17_ctdb_config_delete_ip.sh

@@ -73,5 +73,3 @@ else
     echo "BAD: The remove IP address is still there!"
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/18_ctdb_freeze.sh

@@ -46,7 +46,3 @@ echo "Freezing node $test_node"
 try_command_on_node 0 $CTDB freeze -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
-
-echo "GOOD: That worked!"
-
-ctdb_test_exit

ctdb/tests/simple/19_ctdb_thaw.sh

@@ -53,5 +53,3 @@ echo "That worked!  Now thawing node $test_node"
 try_command_on_node 0 $CTDB thaw -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unfrozen
-
-ctdb_test_exit

ctdb/tests/simple/20_ctdb_getmonmode.sh

@@ -64,7 +64,3 @@ fi
 try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
-
-echo "GOOD: That worked!"
-
-ctdb_test_exit

ctdb/tests/simple/21_ctdb_disablemonitor.sh

@@ -95,7 +95,3 @@ try_command_on_node -v $test_node rm -v "$trigger"
 sleep_for $monitor_interval
 
 onnode 0 $CTDB_TEST_WRAPPER node_has_status $test_node unhealthy
-
-echo "OK, that all worked.  Expect a restart..."
-
-ctdb_test_exit

ctdb/tests/simple/22_ctdb_enablemonitor.sh

@@ -98,7 +98,3 @@ echo "OK: flag file was created so monitoring must be enabled."
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
 
 try_command_on_node -v $test_node ls -l "$detected"
-
-echo "GOOD: That all worked..."
-
-ctdb_test_exit

ctdb/tests/simple/23_ctdb_moveip.sh

@@ -99,7 +99,3 @@ else
     echo "BAD: IP didn't move to ${to_node}."
     exit 1
 fi
-
-echo "GOOD: That worked..."
-
-ctdb_test_exit

ctdb/tests/simple/24_ctdb_getdbmap.sh

@@ -69,7 +69,3 @@ for i in $(seq 1 5) ; do
 	exit 1
     fi
 done
-
-echo "GOOD: That worked..."
-
-ctdb_test_exit

ctdb/tests/simple/25_dumpmemory.sh

@@ -50,5 +50,3 @@ else
     echo "BAD: there not look to be output for all $num_nodes nodes"
     exit 1
 fi    
-
-ctdb_test_exit

ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh

@@ -63,7 +63,3 @@ for i in ip disable enable "ban 0" unban listvars ; do
 	exit 1
     fi
 done
-
-echo "OK, that all worked.  Expect a restart..."
-
-ctdb_test_exit

ctdb/tests/simple/31_ctdb_disable.sh

@@ -66,7 +66,3 @@ else
     echo "Some IPs didn't move."
     testfailures=1
 fi
-
-echo "Expect a restart..."
-
-ctdb_test_exit

ctdb/tests/simple/32_ctdb_enable.sh

@@ -85,7 +85,3 @@ else
     echo "Some IPs didn't move."
     testfailures=1
 fi
-
-echo "All done!"
-
-ctdb_test_exit

ctdb/tests/simple/41_ctdb_ban.sh

@@ -96,5 +96,3 @@ else
     echo "Some IPs didn't move."
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/42_ctdb_unban.sh

@@ -93,5 +93,3 @@ else
     echo "Some IPs didn't move."
     testfailures=1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/51_ctdb_bench.sh

@@ -90,6 +90,3 @@ else
     echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > $check_percent%"
     exit 1
 fi
-
-
-ctdb_test_exit

ctdb/tests/simple/52_ctdb_fetch.sh

@@ -89,5 +89,3 @@ else
     echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
     exit 1
 fi
-
-ctdb_test_exit

ctdb/tests/simple/53_ctdb_transaction.sh

@@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=30"
 
 echo "Running ctdb_transaction on all $num_nodes nodes."
 try_command_on_node -v -pq all "$t & $t"
-
-ctdb_test_exit

ctdb/tests/simple/61_ctdb_persistent_safe.sh

@@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --timelimit=30"
 
 echo "Running ctdb_persistent on all $num_nodes nodes."
 try_command_on_node -v -pq all "$t & $t"
-
-ctdb_test_exit

ctdb/tests/simple/62_ctdb_persistent_unsafe.sh

@@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --unsafe-writes --timelimit=30"
 
 echo "Running ctdb_persistent --unsafe-writes on all $num_nodes nodes."
 try_command_on_node -v -pq all "$t & $t"
-
-ctdb_test_exit