diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 210e6c4853c..6485868cc2d 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -60,12 +60,20 @@ ctdb_test_exit () trap - 0 [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + status=$(($testfailures+0)) + + # Avoid making a test fail from this point onwards. The test is + # now complete. + set +e + + echo "*** TEST COMPLETE (RC=$status), CLEANING UP..." eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook - if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then - echo "Restarting ctdb on all nodes to get back into known state..." + if $ctdb_test_restart_scheduled || \ + ! onnode 0 CTDB_TEST_CLEANING_UP=1 $CTDB_TEST_WRAPPER cluster_is_healthy ; then + restart_ctdb else # This could be made unconditional but then we might get @@ -76,7 +84,7 @@ ctdb_test_exit () onnode 0 ctdb recover fi - test_exit + exit $status } ctdb_test_exit_hook_add () @@ -149,6 +157,7 @@ ctdb_test_init () { scriptname=$(basename "$0") testfailures=0 + ctdb_test_restart_scheduled=false ctdb_test_cmd_options $@ @@ -309,6 +318,14 @@ cluster_is_healthy () exit 0 else echo "Cluster is UNHEALTHY" + if [ -z "$CTDB_TEST_CLEANING_UP" ] ; then + echo "DEBUG:" + local i + for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + echo "$i" + $i || true + done + fi exit 1 fi } @@ -591,9 +608,11 @@ setup_ctdb () restart_ctdb () { - if [ "$1" = "-v" ] ; then - echo "Restarting CTDB (scheduled)..." + echo -n "Restarting CTDB" + if $ctdb_test_restart_scheduled ; then + echo -n " (scheduled)" fi + echo "..." if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop @@ -621,11 +640,9 @@ restart_ctdb () ctdb_restart_when_done () { - ctdb_test_exit_hook_add restart_ctdb -v + ctdb_test_restart_scheduled=true } - - ####################################### install_eventscript () diff --git a/ctdb/tests/simple/00_ctdb_init.sh b/ctdb/tests/simple/00_ctdb_init.sh index 86296f18ff5..7b05b069c07 100755 --- a/ctdb/tests/simple/00_ctdb_init.sh +++ b/ctdb/tests/simple/00_ctdb_init.sh @@ -28,6 +28,5 @@ ctdb_test_init "$@" set -e -echo "Restarting ctdb on all nodes..." setup_ctdb restart_ctdb diff --git a/ctdb/tests/simple/01_ctdb_version.sh b/ctdb/tests/simple/01_ctdb_version.sh index 1cc506fbc00..69a861b7217 100755 --- a/ctdb/tests/simple/01_ctdb_version.sh +++ b/ctdb/tests/simple/01_ctdb_version.sh @@ -48,5 +48,3 @@ else echo "BAD: CTDB version != RPM version" testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh index ddd77f71f85..935e676b925 100755 --- a/ctdb/tests/simple/02_ctdb_listvars.sh +++ b/ctdb/tests/simple/02_ctdb_listvars.sh @@ -39,5 +39,3 @@ sanity_check_output \ 5 \ '^[[:alpha:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \ "$out" - -ctdb_test_exit diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh index cae0f6984bc..0ff0eef4198 100755 --- a/ctdb/tests/simple/03_ctdb_getvar.sh +++ b/ctdb/tests/simple/03_ctdb_getvar.sh @@ -49,7 +49,3 @@ while read var x val ; do exit 1 fi done - -testfailures=$? - -ctdb_test_exit diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh index ceb6c2361d5..336eb392876 100755 --- a/ctdb/tests/simple/04_ctdb_setvar.sh +++ b/ctdb/tests/simple/04_ctdb_setvar.sh @@ -77,7 +77,3 @@ fi echo "Look's good! Putting the old value back..." cmd="$CTDB setvar $var $val" try_command_on_node 0 $cmd - -echo "All done..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh index 6e6af1f8b15..4e4df5d0ffc 100755 --- a/ctdb/tests/simple/05_ctdb_listnodes.sh +++ b/ctdb/tests/simple/05_ctdb_listnodes.sh @@ -57,5 +57,3 @@ while [ $n -lt $num_nodes ] ; do fi n=$(($n + 1)) done - -ctdb_test_exit diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh index 9759df1cd9b..3989d00c795 100755 --- a/ctdb/tests/simple/06_ctdb_getpid.sh +++ b/ctdb/tests/simple/06_ctdb_getpid.sh @@ -82,5 +82,3 @@ while [ $n -lt $num_nodes ] ; do fi n=$(($n + 1)) done <<<"$pids_onnode" - -ctdb_test_exit diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh index 01c186c87ba..da896714ce5 100755 --- a/ctdb/tests/simple/07_ctdb_process_exists.sh +++ b/ctdb/tests/simple/07_ctdb_process_exists.sh @@ -63,16 +63,4 @@ try_command_on_node $test_node 'echo $$' pid="$out" echo "Checking for PID $pid on node $test_node" -# set -e is good, but avoid it here -status=0 -onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$? -echo "$out" - -if [ $status -ne 0 ] ; then - echo "OK" -else - echo "BAD" - testfailures=1 -fi - -ctdb_test_exit +try_command_on_node -v 0 "! $CTDB process-exists ${test_node}:${pid}" diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh index f5f9c6c077f..1704032fc4b 100755 --- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh +++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh @@ -54,5 +54,3 @@ else echo "BAD, there are only ${num_not_rm_lines} nodes claiming not to be the recmaster" testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh index bd23f77b59c..bc315dfee0e 100755 --- a/ctdb/tests/simple/09_ctdb_ping.sh +++ b/ctdb/tests/simple/09_ctdb_ping.sh @@ -52,7 +52,3 @@ sanity_check_output \ 1 \ "(: ctdb_control error: 'ctdb_control to disconnected node'|Unable to get ping response from node 1|Node 1 is DISCONNECTED)" \ "$out" - -echo "Expect a restart..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh index cad89866b17..c05526cd21a 100755 --- a/ctdb/tests/simple/11_ctdb_ip.sh +++ b/ctdb/tests/simple/11_ctdb_ip.sh @@ -59,5 +59,3 @@ else echo "$out" testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh index 5c74bc82af0..05f49caad86 100755 --- a/ctdb/tests/simple/12_ctdb_getdebug.sh +++ b/ctdb/tests/simple/12_ctdb_getdebug.sh @@ -38,7 +38,7 @@ getdebug_onnode="$out" sanity_check_output \ $num_nodes \ - '^Node [[:digit:]]+ is at debug level [[:alpha:]]+ \([[:digit:]]\)$' \ + '^Node [[:digit:]]+ is at debug level [[:alpha:]]+ \([[:digit:]]+\)$' \ "$out" try_command_on_node -v 1 "$CTDB getdebug -n all" @@ -65,7 +65,7 @@ colons="" nl=" " while read line ; do - t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]])\)$@:\1:\2:@') + t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]]+)\)$@:\1:\2:@') colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}" done <<<"$getdebug_onnode" @@ -80,5 +80,3 @@ else echo "$out" testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/13_ctdb_setdebug.sh b/ctdb/tests/simple/13_ctdb_setdebug.sh index dec19e17ab5..a0e0485778f 100755 --- a/ctdb/tests/simple/13_ctdb_setdebug.sh +++ b/ctdb/tests/simple/13_ctdb_setdebug.sh @@ -47,7 +47,7 @@ get_debug () try_command_on_node -v $node "$CTDB getdebug" check_debug=$(echo "$out" | - sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \(-?[[:digit:]]\)$@\1@') + sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \(-?[[:digit:]]+\)$@\1@') } set_and_check_debug () @@ -81,5 +81,3 @@ if [ "$testfailures" != 1 ] ; then echo "Returning the debug level to its initial value..." set_and_check_debug 1 "$initial_debug" fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh index d92cb41cd6c..257ca4a3ca5 100755 --- a/ctdb/tests/simple/14_ctdb_statistics.sh +++ b/ctdb/tests/simple/14_ctdb_statistics.sh @@ -42,5 +42,3 @@ sanity_check_output 40 "$pattern" "$out" try_command_on_node -v 1 "$CTDB statistics -n all" sanity_check_output 40 "$pattern" "$out" - -ctdb_test_exit diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh index 9de83c011f5..5df5c079300 100755 --- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh +++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh @@ -81,5 +81,3 @@ while [ $n -lt $num_nodes ] ; do n=$(($n + 1)) done - -ctdb_test_exit diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh index 634a39aec51..7eacd121878 100755 --- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh +++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh @@ -114,5 +114,3 @@ else echo "BAD: Unable to find IP address to add." testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh index bab45ddd50c..6cda8c17f21 100755 --- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh +++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh @@ -73,5 +73,3 @@ else echo "BAD: The remove IP address is still there!" testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/18_ctdb_freeze.sh b/ctdb/tests/simple/18_ctdb_freeze.sh index 009e2a745c6..6e13139f53f 100755 --- a/ctdb/tests/simple/18_ctdb_freeze.sh +++ b/ctdb/tests/simple/18_ctdb_freeze.sh @@ -46,7 +46,3 @@ echo "Freezing node $test_node" try_command_on_node 0 $CTDB freeze -n $test_node onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen - -echo "GOOD: That worked!" - -ctdb_test_exit diff --git a/ctdb/tests/simple/19_ctdb_thaw.sh b/ctdb/tests/simple/19_ctdb_thaw.sh index c764205911e..86b13937556 100755 --- a/ctdb/tests/simple/19_ctdb_thaw.sh +++ b/ctdb/tests/simple/19_ctdb_thaw.sh @@ -53,5 +53,3 @@ echo "That worked! Now thawing node $test_node" try_command_on_node 0 $CTDB thaw -n $test_node onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unfrozen - -ctdb_test_exit diff --git a/ctdb/tests/simple/20_ctdb_getmonmode.sh b/ctdb/tests/simple/20_ctdb_getmonmode.sh index 6a55341297d..76c5391f906 100755 --- a/ctdb/tests/simple/20_ctdb_getmonmode.sh +++ b/ctdb/tests/simple/20_ctdb_getmonmode.sh @@ -64,7 +64,3 @@ fi try_command_on_node -v 0 $CTDB disablemonitor -n $test_node onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff - -echo "GOOD: That worked!" - -ctdb_test_exit diff --git a/ctdb/tests/simple/21_ctdb_disablemonitor.sh b/ctdb/tests/simple/21_ctdb_disablemonitor.sh index 23ae052f5cb..f9b31a3254b 100755 --- a/ctdb/tests/simple/21_ctdb_disablemonitor.sh +++ b/ctdb/tests/simple/21_ctdb_disablemonitor.sh @@ -95,7 +95,3 @@ try_command_on_node -v $test_node rm -v "$trigger" sleep_for $monitor_interval onnode 0 $CTDB_TEST_WRAPPER node_has_status $test_node unhealthy - -echo "OK, that all worked. Expect a restart..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/22_ctdb_enablemonitor.sh b/ctdb/tests/simple/22_ctdb_enablemonitor.sh index 98fc7e2700e..3f0b082d5bd 100755 --- a/ctdb/tests/simple/22_ctdb_enablemonitor.sh +++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh @@ -98,7 +98,3 @@ echo "OK: flag file was created so monitoring must be enabled." onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval try_command_on_node -v $test_node ls -l "$detected" - -echo "GOOD: That all worked..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh index ca7fe11fb4c..a78ab5e182c 100755 --- a/ctdb/tests/simple/23_ctdb_moveip.sh +++ b/ctdb/tests/simple/23_ctdb_moveip.sh @@ -99,7 +99,3 @@ else echo "BAD: IP didn't move to ${to_node}." exit 1 fi - -echo "GOOD: That worked..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh index c621a304ad6..224cb2a2b2f 100755 --- a/ctdb/tests/simple/24_ctdb_getdbmap.sh +++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh @@ -69,7 +69,3 @@ for i in $(seq 1 5) ; do exit 1 fi done - -echo "GOOD: That worked..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh index 2b719657ccf..d499599fdfa 100755 --- a/ctdb/tests/simple/25_dumpmemory.sh +++ b/ctdb/tests/simple/25_dumpmemory.sh @@ -50,5 +50,3 @@ else echo "BAD: there not look to be output for all $num_nodes nodes" exit 1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh index 4287cba1fbe..fd1e1e68020 100755 --- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh +++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh @@ -63,7 +63,3 @@ for i in ip disable enable "ban 0" unban listvars ; do exit 1 fi done - -echo "OK, that all worked. Expect a restart..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/31_ctdb_disable.sh b/ctdb/tests/simple/31_ctdb_disable.sh index bba03b14cef..98e52f59dd1 100755 --- a/ctdb/tests/simple/31_ctdb_disable.sh +++ b/ctdb/tests/simple/31_ctdb_disable.sh @@ -66,7 +66,3 @@ else echo "Some IPs didn't move." testfailures=1 fi - -echo "Expect a restart..." - -ctdb_test_exit diff --git a/ctdb/tests/simple/32_ctdb_enable.sh b/ctdb/tests/simple/32_ctdb_enable.sh index 7f3da4cd000..5e189de37d3 100755 --- a/ctdb/tests/simple/32_ctdb_enable.sh +++ b/ctdb/tests/simple/32_ctdb_enable.sh @@ -85,7 +85,3 @@ else echo "Some IPs didn't move." testfailures=1 fi - -echo "All done!" - -ctdb_test_exit diff --git a/ctdb/tests/simple/41_ctdb_ban.sh b/ctdb/tests/simple/41_ctdb_ban.sh index e2d36de609e..8299588ed8c 100755 --- a/ctdb/tests/simple/41_ctdb_ban.sh +++ b/ctdb/tests/simple/41_ctdb_ban.sh @@ -96,5 +96,3 @@ else echo "Some IPs didn't move." testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/42_ctdb_unban.sh b/ctdb/tests/simple/42_ctdb_unban.sh index 35085b2a4a2..dc3464d194c 100755 --- a/ctdb/tests/simple/42_ctdb_unban.sh +++ b/ctdb/tests/simple/42_ctdb_unban.sh @@ -93,5 +93,3 @@ else echo "Some IPs didn't move." testfailures=1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/51_ctdb_bench.sh b/ctdb/tests/simple/51_ctdb_bench.sh index 0c9f5c91ed4..75c63a772ae 100755 --- a/ctdb/tests/simple/51_ctdb_bench.sh +++ b/ctdb/tests/simple/51_ctdb_bench.sh @@ -90,6 +90,3 @@ else echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > $check_percent%" exit 1 fi - - -ctdb_test_exit diff --git a/ctdb/tests/simple/52_ctdb_fetch.sh b/ctdb/tests/simple/52_ctdb_fetch.sh index 6b31a21b557..fbc49cda683 100755 --- a/ctdb/tests/simple/52_ctdb_fetch.sh +++ b/ctdb/tests/simple/52_ctdb_fetch.sh @@ -89,5 +89,3 @@ else echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%" exit 1 fi - -ctdb_test_exit diff --git a/ctdb/tests/simple/53_ctdb_transaction.sh b/ctdb/tests/simple/53_ctdb_transaction.sh index 5ea5a471693..9189414c408 100755 --- a/ctdb/tests/simple/53_ctdb_transaction.sh +++ b/ctdb/tests/simple/53_ctdb_transaction.sh @@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=30" echo "Running ctdb_transaction on all $num_nodes nodes." try_command_on_node -v -pq all "$t & $t" - -ctdb_test_exit diff --git a/ctdb/tests/simple/61_ctdb_persistent_safe.sh b/ctdb/tests/simple/61_ctdb_persistent_safe.sh index 26026c9dee0..845d180d807 100755 --- a/ctdb/tests/simple/61_ctdb_persistent_safe.sh +++ b/ctdb/tests/simple/61_ctdb_persistent_safe.sh @@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --timelimit=30" echo "Running ctdb_persistent on all $num_nodes nodes." try_command_on_node -v -pq all "$t & $t" - -ctdb_test_exit diff --git a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh index 6bd6835307e..2a7e3abcd1f 100755 --- a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh +++ b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh @@ -37,5 +37,3 @@ t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --unsafe-writes --timelimit=30" echo "Running ctdb_persistent --unsafe-writes on all $num_nodes nodes." try_command_on_node -v -pq all "$t & $t" - -ctdb_test_exit