2025-01-26 10:04:02 +03:00 · 2009-02-18 13:10:03 +11:00 · 2009-02-18 13:10:03 +11:00 · 5c7570b103
commit 5c7570b103
parent be926a4640 1b61a149f4
66 changed files with 3535 additions and 289 deletions
--- a/ctdb/.gitignore
+++ b/ctdb/.gitignore
@ -15,5 +15,9 @@ utils/smnotify/gen_smnotify.c
 utils/smnotify/gen_xdr.c
 utils/smnotify/smnotify.h
 nodes.txt
+public_addresses.txt
 rec.lock
 test.db
+tests/bin
+tests/events.d/00.ctdb_test_trigger
+tests/var
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@ -56,14 +56,15 @@ CTDB_SERVER_OBJ = server/ctdbd.o server/ctdb_daemon.o server/ctdb_lockwait.o \
 	server/ctdb_keepalive.o server/ctdb_logging.o server/ctdb_uptime.c \
 	$(CTDB_CLIENT_OBJ) $(CTDB_TCP_OBJ) @INFINIBAND_WRAPPER_OBJ@

-TEST_BINS=bin/ctdb_bench bin/ctdb_fetch bin/ctdb_store bin/ctdb_randrec bin/ctdb_persistent \
-	bin/ctdb_traverse bin/rb_test bin/ctdb_transaction \
+TEST_BINS=tests/bin/ctdb_bench tests/bin/ctdb_fetch tests/bin/ctdb_store \
+	tests/bin/ctdb_randrec tests/bin/ctdb_persistent \
+	tests/bin/ctdb_traverse tests/bin/rb_test tests/bin/ctdb_transaction \
 	@INFINIBAND_BINS@

 BINS = bin/ctdb @CTDB_SCSI_IO@ bin/ctdb_ipmux bin/smnotify
 SBINS = bin/ctdbd

-DIRS = lib bin
+DIRS = lib bin tests/bin

 .SUFFIXES: .c .o .h .1 .1.xml .1.html

@ -114,39 +115,39 @@ utils/smnotify/gen_smnotify.c: utils/smnotify/smnotify.x utils/smnotify/smnotify
 	@echo Generating $@
 	rpcgen -l utils/smnotify/smnotify.x > utils/smnotify/gen_smnotify.c 

-bin/rb_test: $(CTDB_CLIENT_OBJ) tests/rb_test.o 
+tests/bin/rb_test: $(CTDB_CLIENT_OBJ) tests/src/rb_test.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/ctdb_bench.o 
+tests/bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/src/ctdb_bench.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/ctdb_fetch.o 
+tests/bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/src/ctdb_fetch.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/ctdb_store.o 
+tests/bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/src/ctdb_store.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/ctdb_traverse.o 
+tests/bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/src/ctdb_traverse.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/ctdb_randrec.o 
+tests/bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/src/ctdb_randrec.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/ctdb_persistent.o 
+tests/bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/src/ctdb_persistent.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/ctdb_transaction.o 
+tests/bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/src/ctdb_transaction.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

-bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
+tests/bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
 	@echo Linking $@
 	@$(CC) $(CFLAGS) -o $@ ib/ibwrapper_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)

--- a/ctdb/ib/config.m4
+++ b/ctdb/ib/config.m4
@ -9,7 +9,7 @@ if eval "test x$enable_infiniband = xyes"; then

 	INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o ib/ibw_ctdb.o ib/ibw_ctdb_init.o"
 	INFINIBAND_LIBS="-lrdmacm -libverbs"
-	INFINIBAND_BINS="bin/ibwrapper_test"
+	INFINIBAND_BINS="tests/bin/ibwrapper_test"

 	AC_CHECK_HEADERS(infiniband/verbs.h, [], [
 		echo "ERROR: you need infiniband/verbs.h when ib enabled!"
--- a/ctdb/tests/README
+++ b/ctdb/tests/README
@ -0,0 +1,106 @@
+Introduction
+------------
+
+The run_tests script can be run as follows:
+
+  scripts/run_tests simple/*.sh
+
+It can also be run from other places (e.g. the top level ctdb source
+directory), as it works out where the tree is.
+
+The pseudo-test simple/00_ctdb_init.sh causes ctdb to be (re)started
+on all nodes to attempt to force the cluster into a healthy state.  By
+default (i.e. if CTDB_TEST_REAL_CLUSTER is unset - see below) this
+causes some local daemons to be started on the local machine.  Tests
+can also be run against a real or virtual cluster.  All tests
+communicate with cluster nodes using onnode - when using local daemons
+this is accomplished via some test hooks in onnode and the ctdb
+client.
+
+Command-line options
+--------------------
+
+The most useful option is "-s", which causes a summary of test results
+to be printed at the end of testing.
+
+Environment variables
+---------------------
+
+run_tests supports several environment variables, mostly implemented
+in scripts/ctdb_test_env.  These are:
+
+* CTDB_TEST_REAL_CLUSTER
+
+  If set, testing will occur on a real or virtual cluster.
+
+  Assumptions:
+
+  - The ctdb client command can be found via $PATH on the nodes.
+
+  - Password-less ssh access to the cluster nodes is permitted from
+    the test host.
+
+  - $CTDB_NODES_FILE is set to the location of a file similar to
+    /etc/ctdb/nodes.  The file can be obtained by scping it from one
+    of the cluster nodes.
+
+  - See CTDB_TEST_REMOTE_DIR.
+
+  If not set, testing will proceed against local daemons.
+
+* CTDB_TEST_REMOTE_DIR
+
+  This may be required when running against a real or virtual cluster
+  (as opposed to local daemons).
+
+  If set, this points to a directory containing the contents of the
+  tests/scripts/ directory, as well as all of the test binaries.  This
+  can be accomplished in a couple of ways:
+
+  * By copying the relevant scripts/binaries to some directory.
+
+  * Building an RPM containing all of the test code that is required
+    on the cluster nodes and installing it on each node.  Hopefully
+    this will be supported in a future version of the CTDB packaging
+    process.
+
+  If not set, the test system assumes that the CTDB tree is available
+  in the same location on the cluster nodes as on the test host.  This
+  could be accomplished by copying or by sharing with NFS (or
+  similar).
+
+* VALGRIND
+
+  This can be used to cause all invocations of the ctdb client (and,
+  with local daemons, the ctdbd daemons themselves) to occur under
+  valgrind.
+
+  The easiest way of doing this is something like:
+
+    VALGRIND="valgrind -q" scripts/run_tests ...
+
+  NOTE: Some libc calls seem to do weird things and perhaps cause
+  spurious output from ctdbd at start time.  Please read valgrind
+  output carefully before reporting bugs.  :-)
+
+* CTDB
+
+  How to invoke the ctdb client.  If not already set and if $VALGRIND
+  is set, this is set to "$VALGRIND ctdb".  If this is not already set
+  but $VALGRIND is not set, this is simply set to "ctdb"
+
+Look, no run_test!
+------------------
+
+If you want to integrate individual tests into some other test
+environment you can use scripts/ctdb_test_env to wrap individual
+tests.  They will return 0 on success, non-zero otherwise, and will
+print the output omitting the test header/footer that surrounds test
+output when tests are run using run_tests.  So, you can do something
+like:
+
+  for i in simple/*.sh ; do
+      my_test_framework_wrapper scripts/ctdb_test_env $i
+  done
+
+to have your own framework process the test results and output.
--- a/ctdb/tests/TODO
+++ b/ctdb/tests/TODO
@ -0,0 +1,4 @@
+* Make tests know about IPv6.
+* Tests that write to database.
+* Tests that check actual network connectivity on failover.
+* Handle interrupting tests better.
--- a/ctdb/tests/bench.sh
+++ b/ctdb/tests/bench.sh
@ -1,24 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_bench ctdbd
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
-    NUMNODES=$1
-fi
-
-rm -f nodes.txt
-for i in `seq 1 $NUMNODES`; do
-  echo 127.0.0.$i >> nodes.txt
-done
-
-tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
-
-killall -9 ctdb_bench
-echo "Trying $NUMNODES nodes"
-for i in `seq 1 $NUMNODES`; do
-  $VALGRIND bin/ctdb_bench --socket sock.$i -n $NUMNODES $*  &
-done
-
-wait
-bin/ctdb shutdown --socket sock.1 -n all
--- a/ctdb/tests/ctdbd.sh
+++ b/ctdb/tests/ctdbd.sh
@ -1,50 +0,0 @@
-#!/bin/sh
-
-killall -q ctdbd
-
-tests/start_daemons.sh 2 || exit 1
-
-echo "Testing ping"
-$VALGRIND bin/ctdb ping || exit 1
-
-echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
-
-echo "Testing statistics"
-$VALGRIND bin/ctdb -n all statistics || exit 1
-
-echo "Testing statisticsreset"
-$VALGRIND bin/ctdb -n all statisticsreset || exit 1
-
-echo "Testing debug"
-$VALGRIND bin/ctdb -n all setdebug 3 || exit 1
-$VALGRIND bin/ctdb -n all getdebug || exit 1
-$VALGRIND bin/ctdb -n all setdebug 0 || exit 1
-$VALGRIND bin/ctdb -n all getdebug || exit 1
-
-echo "Attaching to some databases"
-$VALGRIND bin/ctdb attach test1.tdb || exit 1
-$VALGRIND bin/ctdb attach test2.tdb || exit 1
-
-echo "Testing getdbmap"
-$VALGRIND bin/ctdb getdbmap || exit 1
-
-echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
-
-echo "Testing variables"
-$VALGRIND bin/ctdb listvars || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
-$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
-
-sleep 1
-
-echo "Testing shutdown"
-$VALGRIND bin/ctdb shutdown -n all || exit 1
-
-sleep 1
-
-echo "All done"
-killall -q ctdbd
-exit 0
--- a/ctdb/tests/fetch.sh
+++ b/ctdb/tests/fetch.sh
@ -1,21 +0,0 @@
-#!/bin/sh
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
-    NUMNODES=$1
-fi
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_fetch; exit 1' INT TERM
-
-tests/start_daemons.sh $NUMNODES || exit 1
-
-
-killall -9 -q ctdb_fetch
-for i in `seq 1 $NUMNODES`; do
-  $VALGRIND bin/ctdb_fetch --socket sock.$i -n $NUMNODES $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-exit 0
--- a/ctdb/tests/nodes.txt
+++ b/ctdb/tests/nodes.txt
@ -1,4 +0,0 @@
-127.0.0.1
-127.0.0.2
-127.0.0.3
-127.0.0.4
--- a/ctdb/tests/persistent.sh
+++ b/ctdb/tests/persistent.sh
@ -1,47 +0,0 @@
-#!/bin/sh
-
-NUMNODES=4
-if [ $# -gt 0 ]; then
-    NUMNODES=$1
-fi
-
-killall -9 -q ctdb_persistent ctdbd
-
-rm -rf test.db/persistent
-
-echo "Starting $NUMNODES daemons for SAFE persistent writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_persistent; exit 1' INT TERM
-
-
-for i in `seq 1 $NUMNODES`; do
-  $VALGRIND bin/ctdb_persistent --timelimit 30 --socket sock.$i $* &
-  $VALGRIND bin/ctdb_persistent --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-
-
-echo "Starting $NUMNODES daemons for UNSAFE persistent writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-killall -9 -q ctdb_persistent
-
-for i in `seq 1 $NUMNODES`; do
-  $VALGRIND bin/ctdb_persistent --unsafe-writes --timelimit 30 --socket sock.$i $* &
-  $VALGRIND bin/ctdb_persistent --unsafe-writes --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-
-
-exit 0
--- a/ctdb/tests/public_addresses
+++ b/ctdb/tests/public_addresses
@ -1,4 +0,0 @@
-10.99.99.0/24
-10.99.99.1/24
-10.99.99.2/24
-10.99.99.3/24
--- a/ctdb/tests/run_tests.sh
+++ b/ctdb/tests/run_tests.sh
@ -1,10 +1,6 @@
 #!/bin/sh

-trap 'echo "Killing test"; killall -9 -q ctdbd; exit 1' INT TERM
-
-tests/fetch.sh 4 || exit 1
-tests/bench.sh 4 || exit 1
-tests/ctdbd.sh || exit 1
+tests/scripts/run_tests -s tests/simple/*.sh || exit 1

 echo "All OK"
 exit 0
--- a/ctdb/tests/scripts/ctdb_test_env
+++ b/ctdb/tests/scripts/ctdb_test_env
@ -0,0 +1,43 @@
+#!/bin/bash
+
+ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd)
+export CTDB_DIR=$(dirname $(dirname $ctdb_test_scripts_dir))
+var_dir=$CTDB_DIR/tests/var
+
+######################################################################
+
+ctdb_tools_dir=$CTDB_DIR/tools
+
+PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}"
+
+export CTDB_TIMEOUT=60
+
+######################################################################
+
+if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then
+    CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap"
+else
+    CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap"
+fi
+export CTDB_TEST_WRAPPER
+
+# If we're not running on a real cluster then we need a local copy of
+# ctdb (and other stuff) in $PATH and we will use local daemons.
+if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+    export CTDB_TEST_NUM_DAEMONS=3
+
+    export CTDB_NODES_SOCKETS=""
+    for i in $(seq 0 $(($CTDB_TEST_NUM_DAEMONS -1))) ; do
+	CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}"
+    done
+
+    PATH="${CTDB_DIR}/bin:${CTDB_DIR}/tests/bin:${PATH}"
+fi
+
+# If $VALGRIND is set then use it whenever ctdb is called, but only if
+# $CTDB is not already set.
+[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb"
+
+######################################################################
+
+"$@"
--- a/ctdb/tests/scripts/ctdb_test_functions.bash
+++ b/ctdb/tests/scripts/ctdb_test_functions.bash
@ -0,0 +1,577 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+fail ()
+{
+    echo "$*"
+    exit 1
+}
+
+######################################################################
+
+ctdb_test_begin ()
+{
+    local name="$1"
+
+    teststarttime=$(date '+%s')
+    testduration=0
+
+    echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--"
+    echo "Running test $name ($(date '+%T'))"
+    echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--"
+}
+
+ctdb_test_end ()
+{
+    local name="$1" ; shift
+    local status="$1" ; shift
+    # "$@" is command-line
+
+    local interp="SKIPPED"
+    local statstr=" (reason $*)"
+    if [ -n "$status" ] ; then
+	if [ $status -eq 0 ] ; then
+	    interp="PASSED"
+	    statstr=""
+	    echo "ALL OK: $*"
+	else
+	    interp="FAILED"
+	    statstr=" (status $status)"
+	    testfailures=$(($testfailures+1))
+	fi
+    fi
+
+    testduration=$(($(date +%s)-$teststarttime))
+
+    echo "=========================================================================="
+    echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)"
+    echo "=========================================================================="
+
+}
+
+test_exit ()
+{
+    exit $(($testfailures+0))
+}
+
+ctdb_test_exit ()
+{
+    local status=$?
+
+    trap - 0
+
+    [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
+
+    eval "$ctdb_test_exit_hook"
+    unset ctdb_test_exit_hook
+
+    if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
+	echo "Restarting ctdb on all nodes to get back into known state..."
+	restart_ctdb
+    else
+	# This could be made unconditional but then we might get
+	# duplication from the recovery in restart_ctdb.  We want to
+	# leave the recovery in restart_ctdb so that future tests that
+	# might do a manual restart mid-test will benefit.
+	echo "Forcing a recovery..."
+	onnode 0 ctdb recover
+    fi
+
+    test_exit
+}
+
+ctdb_test_run ()
+{
+    local name="$1" ; shift
+    
+    [ -n "$1" ] || set -- "$name"
+
+    ctdb_test_begin "$name"
+
+    local status=0
+    "$@" || status=$?
+
+    ctdb_test_end "$name" "$status" "$*"
+    
+    return $status
+}
+
+ctdb_test_usage()
+{
+    local status=${1:-2}
+    
+    cat <<EOF
+Usage: $0 [option]
+
+Options:	
+    -h, --help          show this screen.
+    -v, --version       show test case version.
+    --category          show the test category (ACL, CTDB, Samba ...).
+    -d, --description   show test case description.
+    --summary           show short test case summary.
+EOF
+
+    exit $status
+}
+
+ctdb_test_version ()
+{
+    [ -n "$CTDB_DIR" ] || fail "Can not determine version."
+
+    (cd "$CTDB_DIR" && git describe)
+}
+
+ctdb_test_cmd_options()
+{
+    [ -n "$1" ] || return 0
+
+    case "$1" in
+        -h|--help)        ctdb_test_usage 0   ;;
+        -v|--version)     ctdb_test_version   ;;
+        --category)       echo "CTDB"         ;; 
+        -d|--description) test_info           ;;
+	--summary)        test_info | head -1 ;;
+	*)
+	    echo "Error: Unknown parameter = $1"
+	    echo
+	    ctdb_test_usage 2
+	    ;;
+    esac
+
+    exit 0
+}
+
+ctdb_test_init () 
+{
+    scriptname=$(basename "$0")
+    testfailures=0
+
+    ctdb_test_cmd_options $@
+
+    trap "ctdb_test_exit" 0
+}
+
+########################################
+
+# Sets: $out
+try_command_on_node ()
+{
+    local nodespec="$1" ; shift
+
+    local verbose=false
+    local onnode_opts=""
+
+    while [ "${nodespec#-}" != "$nodespec" ] ; do
+	if [ "$nodespec" = "-v" ] ; then
+	    verbose=true
+	else
+	    onnode_opts="$nodespec"
+	fi
+	nodespec="$1" ; shift
+    done
+
+    local cmd="$*"
+
+    out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
+
+	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
+	echo "$out"
+	return 1
+    }
+
+    if $verbose ; then
+	echo "Output of \"$cmd\":"
+	echo "$out"
+    fi
+}
+
+sanity_check_output ()
+{
+    local min_lines="$1"
+    local regexp="$2" # Should be anchored as necessary.
+    local output="$3"
+
+    local ret=0
+
+    local num_lines=$(echo "$output" | wc -l)
+    echo "There are $num_lines lines of output"
+    if [ $num_lines -lt $min_lines ] ; then
+	echo "BAD: that's less than the required number (${min_lines})"
+	ret=1
+    fi
+
+    local status=0
+    local unexpected # local doesn't pass through status of command on RHS.
+    unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
+
+    # Note that this is reversed.
+    if [ $status -eq 0 ] ; then
+	echo "BAD: unexpected lines in output:"
+	echo "$unexpected" | cat -A
+	ret=1
+    else
+	echo "Output lines look OK"
+    fi
+
+    return $ret
+}
+
+sanity_check_ips ()
+{
+    local ips="$1" # Output of "ctdb ip -n all"
+
+    echo "Sanity checking IPs..."
+
+    local x ipp prev
+    prev=""
+    while read x ipp ; do
+	[ "$ipp" = "-1" ] && break
+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+	    echo "OK"
+	    return 0
+	fi
+	prev="$ipp"
+    done <<<"$ips"
+
+    echo "BAD: a node was -1 or IPs are only assigned to one node"
+    echo "Are you running an old version of CTDB?"
+    return 1
+}
+
+#######################################
+
+# Wait until either timeout expires or command succeeds.  The command
+# will be tried once per second.
+wait_until ()
+{
+    local timeout="$1" ; shift # "$@" is the command...
+
+    echo -n "<${timeout}|"
+    while [ $timeout -gt 0 ] ; do
+	if "$@" ; then
+	    echo '|'
+	    echo "OK"
+	    return 0
+	fi
+	echo -n .
+	timeout=$(($timeout - 1))
+	sleep 1
+    done
+    
+    echo "*TIMEOUT*"
+    
+    return 1
+}
+
+sleep_for ()
+{
+    echo -n "=${1}|"
+    for i in $(seq 1 $1) ; do
+	echo -n '.'
+	sleep 1
+    done
+    echo '|'
+}
+
+_cluster_is_healthy ()
+{
+    local out x count line
+
+    out=$(ctdb -Y status 2>&1) || return 1
+
+    {
+        read x
+	count=0
+        while read line ; do
+	    count=$(($count + 1))
+	    [ "${line#:*:*:}" != "0:0:0:0:" ] && return 1
+        done
+	[ $count -gt 0 ] && return $?
+    } <<<"$out" # Yay bash!
+}
+
+cluster_is_healthy ()
+{
+    if _cluster_is_healthy ; then
+	echo "Cluster is HEALTHY"
+	exit 0
+    else
+	echo "Cluster is UNHEALTHY"
+	exit 1
+    fi
+}
+
+wait_until_healthy ()
+{
+    local timeout="${1:-120}"
+
+    echo "Waiting for cluster to become healthy..."
+
+    wait_until 120 _cluster_is_healthy
+}
+
+# This function is becoming nicely overloaded.  Soon it will collapse!  :-)
+node_has_status ()
+{
+    local pnn="$1"
+    local status="$2"
+
+    local bits fpat mpat
+    case "$status" in
+	(unhealthy)    bits="?:?:?:1" ;;
+	(healthy)      bits="?:?:?:0" ;;
+	(disconnected) bits="1:?:?:?" ;;
+	(connected)    bits="0:?:?:?" ;;
+	(banned)       bits="?:1:?:?" ;;
+	(unbanned)     bits="?:0:?:?" ;;
+	(disabled)     bits="?:?:1:?" ;;
+	(enabled)      bits="?:?:0:?" ;;
+	(frozen)       fpat='^[[:space:]]+frozen[[:space:]]+1$' ;;
+	(unfrozen)     fpat='^[[:space:]]+frozen[[:space:]]+0$' ;;
+	(monon)        mpat='^Monitoring mode:ACTIVE \(0\)$' ;;
+	(monoff)       mpat='^Monitoring mode:DISABLED \(1\)$' ;;
+	*)
+	    echo "node_has_status: unknown status \"$status\""
+	    return 1
+    esac
+
+    if [ -n "$bits" ] ; then
+	local out x line
+
+	out=$(ctdb -Y status 2>&1) || return 1
+
+	{
+            read x
+            while read line ; do
+		[ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0
+            done
+	    return 1
+	} <<<"$out" # Yay bash!
+    elif [ -n "$fpat" ] ; then
+	ctdb statistics -n "$pnn" | egrep -q "$fpat"
+    elif [ -n "$mpat" ] ; then
+	ctdb getmonmode -n "$pnn" | egrep -q "$mpat"
+    else
+	echo 'node_has_status: unknown mode, neither $bits nor $fpat is set'
+	return 1
+    fi
+}
+
+wait_until_node_has_status ()
+{
+    local pnn="$1"
+    local status="$2"
+    local timeout="${3:-30}"
+
+    echo "Waiting until node $pnn has status \"$status\"..."
+
+    wait_until $timeout node_has_status "$pnn" "$status"
+}
+
+# Useful for superficially testing IP failover.
+# IPs must be on nodes matching nodeglob.
+ips_are_on_nodeglob ()
+{
+    local nodeglob="$1" ; shift
+    local ips="$*"
+
+    local out
+
+    try_command_on_node 1 ctdb ip -n all
+
+    while read ip pnn ; do
+	for check in $ips ; do
+	    if [ "$check" = "$ip" ] ; then
+		case "$pnn" in
+		    ($nodeglob) : ;;
+		    (*) return 1  ;;
+		esac
+		ips="${ips/${ip}}" # Remove from list
+	    fi
+	done
+    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+    ips="${ips// }" # Remove any spaces.
+    [ -z "$ips" ]
+}
+
+wait_until_ips_are_on_nodeglob ()
+{
+    echo "Waiting for IPs to fail over..."
+
+    wait_until 60 ips_are_on_nodeglob "$@"
+}
+
+#######################################
+
+daemons_stop ()
+{
+    echo "Attempting to politely shutdown daemons..."
+    onnode 1 ctdb shutdown -n all || true
+
+    echo "Sleeping for a while..."
+    sleep_for 1
+
+    if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
+	echo "Killing remaining daemons..."
+	pkill -f $CTDB_DIR/bin/ctdbd
+
+	if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then
+	    echo "Once more with feeling.."
+	    pkill -9 $CTDB_DIR/bin/ctdbd
+	fi
+    fi
+
+    local var_dir=$CTDB_DIR/tests/var
+    rm -rf $var_dir/test.db
+}
+
+daemons_setup ()
+{
+    local num_nodes="${1:-2}" # default is 2 nodes
+
+    local var_dir=$CTDB_DIR/tests/var
+
+    mkdir -p $var_dir/test.db/persistent
+
+    local nodes=$var_dir/nodes.txt
+    local public_addresses=$var_dir/public_addresses.txt
+    local no_public_addresses=$var_dir/no_public_addresses.txt
+    rm -f $nodes $public_addresses $no_public_addresses
+
+    # If there are (strictly) greater than 2 nodes then we'll randomly
+    # choose a node to have no public addresses.
+    local no_public_ips=-1
+    [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes))
+    echo "$no_public_ips" >$no_public_addresses
+
+    local i
+    for i in $(seq 1 $num_nodes) ; do
+	if [ "${CTDB_USE_IPV6}x" != "x" ]; then
+	    echo ::$i >> $nodes
+	    ip addr add ::$i/128 dev lo
+	else
+	    echo 127.0.0.$i >> $nodes
+	    # 2 public addresses on most nodes, just to make things interesting.
+	    if [ $(($i - 1)) -ne $no_public_ips ] ; then
+		echo "192.0.2.$i/24 lo" >> $public_addresses
+		echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses
+	    fi
+	fi
+    done
+}
+
+daemons_start ()
+{
+    local num_nodes="${1:-2}" # default is 2 nodes
+    shift # "$@" gets passed to ctdbd
+
+    local var_dir=$CTDB_DIR/tests/var
+
+    local nodes=$var_dir/nodes.txt
+    local public_addresses=$var_dir/public_addresses.txt
+    local no_public_addresses=$var_dir/no_public_addresses.txt
+
+    local no_public_ips=-1
+    [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses
+
+    local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent"
+
+    echo "Starting $num_nodes ctdb daemons..."
+    if  [ "$no_public_ips" != -1 ] ; then
+	echo "Node $no_public_ips will have no public IPs."
+    fi
+
+    for i in $(seq 0 $(($num_nodes - 1))) ; do
+	if [ $(id -u) -eq 0 ]; then
+            ctdb_options="$ctdb_options --public-interface=lo"
+	fi
+
+	if [ $i -eq $no_public_ips ] ; then
+	    ctdb_options="$ctdb_options --public-addresses=/dev/null"
+	else
+	    ctdb_options="$ctdb_options --public-addresses=$public_addresses"
+	fi
+
+	# Need full path so we can use "pkill -f" to kill the daemons.
+	$VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1
+    done
+
+    if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then 
+	ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1
+    fi
+}
+
+#######################################
+
+_restart_ctdb ()
+{
+    if [ -e /etc/redhat-release ] ; then
+	service ctdb restart
+    else
+	/etc/init.d/ctdb restart
+    fi
+}
+
+setup_ctdb ()
+{
+    if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+	daemons_setup $CTDB_TEST_NUM_DAEMONS
+    fi
+}
+
+restart_ctdb ()
+{
+    if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+	daemons_stop
+	daemons_start $CTDB_TEST_NUM_DAEMONS
+    else
+	onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb 
+    fi || return 1
+	
+    onnode -q 1  $CTDB_TEST_WRAPPER wait_until_healthy || return 1
+
+    echo "Setting RerecoveryTimeout to 1"
+    onnode -pq all "ctdb setvar RerecoveryTimeout 1"
+
+    # In recent versions of CTDB, forcing a recovery like this blocks
+    # until the recovery is complete.  Hopefully this will help the
+    # cluster to stabilise before a subsequent test.
+    echo "Forcing a recovery..."
+    onnode -q 0 ctdb recover
+
+    #echo "Sleeping to allow ctdb to settle..."
+    #sleep_for 10
+
+    echo "ctdb is ready"
+}
+
+#######################################
+
+install_eventscript ()
+{
+    local script_name="$1"
+    local script_contents="$2"
+
+    if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+	# The quoting here is *very* fragile.  However, we do
+	# experience the joy of installing a short script using
+	# onnode, and without needing to know the IP addresses of the
+	# nodes.
+	onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\""
+    else
+	f="${CTDB_DIR}/tests/events.d/${script_name}"
+	echo "$script_contents" >"$f"
+	chmod 755 "$f"
+    fi
+}
+
+uninstall_eventscript ()
+{
+    local script_name="$1"
+
+    if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+	onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\""
+    else
+	rm -vf "${CTDB_DIR}/tests/events.d/${script_name}"
+    fi
+}
--- a/ctdb/tests/scripts/run_tests
+++ b/ctdb/tests/scripts/run_tests
@ -0,0 +1,64 @@
+#!/bin/bash
+
+# The ability of ctdb_test_env to take tests on the command-line is
+# nice, but here we need to hack around it with that colon to reset
+# the arguments that it sees.
+. $(dirname $0)/ctdb_test_env :
+
+. ctdb_test_functions.bash
+
+usage() {
+    cat <<EOF
+Usage: run_tests [OPTIONS] [TESTS]
+
+EOF
+    exit 1
+}
+
+######################################################################
+
+with_summary=false
+
+temp=$(getopt -n "$prog" -o "xhs" -l help -- "$@")
+
+[ $? != 0 ] && usage
+
+eval set -- "$temp"
+
+while true ; do
+    case "$1" in
+	-x) set -x; shift ;;
+	-s) with_summary=true ; shift ;;
+	--) shift ; break ;;
+	*) usage ;;
+    esac
+done
+
+######################################################################
+
+tests_total=0
+tests_passed=0
+summary=""
+
+rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@')
+ww=$((rows - 7))
+
+for f; do
+    [ -x $f ] || fail "test \"$f\" is not executable"
+    tests_total=$(($tests_total + 1))
+    if ctdb_test_run "$f" ; then
+	tests_passed=$(($tests_passed + 1))
+	t="PASSED"
+    else
+	t="FAILED"
+    fi
+    summary=$(printf "%s\n%-${ww}s%s" "$summary" "$f" "$t")
+done
+
+if $with_summary ; then
+    echo "$summary"
+    echo
+    echo "${tests_passed}/${tests_total} tests passed"
+fi
+
+test_exit
--- a/ctdb/tests/scripts/test_wrap
+++ b/ctdb/tests/scripts/test_wrap
@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Execute the given command.  The intention is that it is a function
+# from ctdb_test_functions.bash.
+
+PATH="$(dirname $0):${PATH}"
+
+. ctdb_test_functions.bash
+
+"$@"
--- a/ctdb/tests/simple/00_ctdb_init.sh
+++ b/ctdb/tests/simple/00_ctdb_init.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Restart the ctdbd daemons of a CTDB cluster.
+
+No error if ctdbd is not already running on the cluster.
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. Restart the ctdb daemons on all nodes using a method according to
+   the test environment and platform.
+
+Expected results:
+
+* The cluster is healthy within a reasonable timeframe.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+echo "Restarting ctdb on all nodes..."
+setup_ctdb
+restart_ctdb
--- a/ctdb/tests/simple/00_ctdb_install_eventscript.sh
+++ b/ctdb/tests/simple/00_ctdb_install_eventscript.sh
@ -0,0 +1,58 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Install an event script on all nodes that helps detect ctdb events.
+
+We could install separate scripts for particular events later as
+needed.  However, the script installed here will allow detection of
+all events.  It also allows a file to be created to indicate that a
+node should be marked as unhealthy.
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. Use the install_eventscript to install the eventscript.
+
+Expected results:
+
+* The script is successfully installed on all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+script='#!/bin/sh
+out=$(ctdb pnn)
+pnn="${out#PNN:}"
+
+# Allow creation of flag files that are removed to confirm that events
+# are taking place.
+rm -f "/tmp/ctdb-test-flag.${1}.${pnn}"
+
+# Allow creation of a trigger file to make a monitor event fail and
+# force a node to be marked as unhealthy.  This avoids having to look
+# at log files to confirm that monitoring is working.  Note that
+# ${pnn} is needed in the filename if we are testing using local
+# daemons so we put in there regardless.
+trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}"
+detected="/tmp/ctdb-test-unhealthy-detected.${pnn}"
+if [ "$1" = "monitor" ] ; then
+    if [ -e "$trigger" ] ; then
+        echo "${0}: Unhealthy because \"$trigger\" detected"
+        touch "$detected"
+        exit 1
+    elif [ -e "$detected" -a ! -e "$trigger" ] ; then
+        echo "${0}: Healthy again, \"$trigger\" no longer detected"
+        rm "$detected"
+    fi
+fi
+
+exit 0
+'
+
+install_eventscript "00.ctdb_test_trigger" "$script"
--- a/ctdb/tests/simple/00_ctdb_onnode.sh
+++ b/ctdb/tests/simple/00_ctdb_onnode.sh
@ -0,0 +1,29 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Use 'onnode' to confirm connectivity between all cluster nodes.
+
+Steps:
+
+1. Do a recursive "onnode all" to make sure all the nodes can connect
+   to each other.  On a cluster this ensures that SSH keys are known
+   between all hosts, which will stop output being corrupted with
+   messages about nodes being added to the list of known hosts.
+
+Expected results:
+
+* 'onnode' works between all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+
+# 
+
+echo "Checking connectivity between nodes..."
+onnode all onnode all true
--- a/ctdb/tests/simple/01_ctdb_version.sh
+++ b/ctdb/tests/simple/01_ctdb_version.sh
@ -0,0 +1,52 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the output of the 'ctdb version' command.
+
+This test assumes an RPM-based installation and needs to be skipped on
+non-RPM systems.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb version' command on one of the cluster nodes.
+3. Compare the version displayed with that listed by the rpm command
+   for the ctdb package.
+
+Expected results:
+
+* The 'ctdb version' command displays the ctdb version number.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+if ! try_command_on_node -v 0 "rpm -q ctdb" ; then
+    echo "No useful output from rpm, SKIPPING rest of test".
+    exit 0
+fi
+rpm_ver="${out#ctdb-}"
+
+try_command_on_node -v 0 "$CTDB version"
+ctdb_ver="${out#CTDB version: }"
+
+if [ "$ctdb_ver" = "$rpm_ver" ] ; then
+    echo "OK: CTDB version = RPM version"
+else
+    echo "BAD: CTDB version != RPM version"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/02_ctdb_listvars.sh
+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
@ -0,0 +1,43 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb listvars' shows a list of all tunable variables.
+
+This test simply checks that at least 5 sane looking lines are
+printed.  It does not check that the list is complete or that the
+values are sane.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listvars' and verify that it shows a list of tunable
+   variables and their current values.
+
+Expected results:
+
+* 'ctdb listvars' works as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listvars"
+
+sanity_check_output \
+    5 \
+    '^[[:alpha:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
+    "$out"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/03_ctdb_getvar.sh
+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
@ -0,0 +1,55 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb getvar' works correctly.
+
+Expands on the steps below as it actually checks the values of all
+variables listed by 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getvars <varname>' with a valid variable name (possibly
+   obtained via 'ctdb listvars'.
+3. Verify that the command displays the correct value of the variable
+   (corroborate with the value shown by 'ctdb listvars'.
+
+Expected results:
+
+* 'ctdb getvar' shows the correct value of the variable.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listvars"
+
+echo "Veryifying all variable values using \"ctdb getvar\"..."
+
+echo "$out" |
+while read var x val ; do
+    try_command_on_node 0 "$CTDB getvar $var"
+
+    val2=$(echo $out | sed -e 's@.*[[:space:]]@@')
+
+    if [ "$val" != "$val2" ] ; then
+	echo "MISMATCH on $var: $val != $val2"
+	exit 1
+    fi
+done
+
+testfailures=$?
+
+ctdb_test_exit
--- a/ctdb/tests/simple/04_ctdb_setvar.sh
+++ b/ctdb/tests/simple/04_ctdb_setvar.sh
@ -0,0 +1,81 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb setvar' works correctly.
+
+Doesn't strictly follow the procedure outlines below, since it doesn't
+pick a variable from the output of 'ctdb listvars'.  However, it
+verifies the value with 'ctdb getvar' in addition to 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get a list of all the ctdb tunable variables, using the 'ctdb
+   listvars' command.
+3. Set the value of one of the variables using the 'setvar' control on
+   one of the nodes.  E.g. 'ctdb setvar DeterministicIPs 0'.
+4. Verify that the 'listvars' control now shows the new value for the
+   variable.
+
+Expected results:
+
+* After setting a value using 'ctdb setvar', 'ctdb listvars' shows the
+  modified value of the variable.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+var="RecoverTimeout"
+
+try_command_on_node -v 0 $CTDB getvar $var
+
+val="${out#*= }"
+
+echo "Going to try incrementing it..."
+
+incr=$(($val + 1))
+
+try_command_on_node 0 $CTDB setvar $var $incr
+
+echo "That seemed to work, let's check the value..."
+
+try_command_on_node -v 0 $CTDB getvar $var
+
+newval="${out#*= }"
+
+if [ "$incr" != "$newval" ] ; then
+    echo "Nope, that didn't work..."
+    exit 1
+fi
+
+echo "Look's good!  Now verifying with \"ctdb listvars\""
+try_command_on_node -v 0 "$CTDB listvars | grep '^$var'"
+
+check="${out#*= }"
+
+if [ "$incr" != "$check" ] ; then
+    echo "Nope, that didn't work.  Restarting ctdb to get back into known state..."
+    restart_ctdb
+    exit 1
+fi
+
+echo "Look's good!  Putting the old value back..."
+cmd="$CTDB setvar $var $val"
+try_command_on_node 0 $cmd
+
+echo "All done..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@ -0,0 +1,61 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb listnodes' shows the list of nodes in a ctdb cluster.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listnodes' on all the nodes of the cluster.
+3. Verify that one all the nodes the command displays a list of
+   current cluster nodes.
+
+Expected results:
+
+* 'ctdb listnodes' displays the correct information.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB listnodes"
+
+num_nodes=$(echo "$out" | wc -l)
+
+# Each line should look like an IP address.
+sanity_check_output \
+    2 \
+    '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' \
+    "$out"
+
+out_0="$out"
+
+echo "Checking other nodes..."
+
+n=1
+while [ $n -lt $num_nodes ] ; do
+    echo -n "Node ${n}: "
+    try_command_on_node $n "$CTDB listnodes"
+    if [ "$out_0" = "$out" ] ; then
+	echo "OK"
+    else
+	echo "DIFFERs from node 0:"
+	echo "$out"
+	testfailures=1
+    fi
+    n=$(($n + 1))
+done
+
+ctdb_test_exit
--- a/ctdb/tests/simple/06_ctdb_getpid.sh
+++ b/ctdb/tests/simple/06_ctdb_getpid.sh
@ -0,0 +1,86 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb getpid' works as expected.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getpid -n <number>' on the nodes to check the PID of the
+   ctdbd process.
+3. Verify that the output is valid.
+4. Verify that with the '-n all' option the command shows the PIDs on
+   all the nodes
+
+Expected results:
+
+* 'ctdb getpid' shows valid output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# This is an attempt at being independent of the number of nodes
+# reported by "ctdb getpid -n all".
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+# Call getpid a few different ways and make sure the answer is always the same.
+
+try_command_on_node -v 0 "onnode -q all $CTDB getpid"
+pids_onnode="$out"
+
+try_command_on_node -v 0 "$CTDB getpid -n all"
+pids_getpid_all="$out"
+
+cmd=""
+n=0
+while [ $n -lt $num_nodes ] ; do
+    cmd="${cmd}${cmd:+; }$CTDB getpid -n $n"
+    n=$(($n + 1))
+done
+try_command_on_node -v 0 "( $cmd )"
+pids_getpid_n="$out"
+
+if [ "$pids_onnode" = "$pids_getpid_all" -a \
+    "$pids_getpid_all" = "$pids_getpid_n" ] ; then
+    echo "They're the same... cool!"
+else
+    echo "Error: they differ."
+    testfailures=1
+fi
+
+echo "Checking each PID for validity"
+
+n=0
+while [ $n -lt $num_nodes ] ; do
+    read line
+    pid=${line#Pid:}
+    try_command_on_node $n "ls -l /proc/${pid}/exe | sed -e 's@.*/@@'"
+    echo -n "Node ${n}, PID ${pid} looks to be running \"$out\" - "
+    if [ "$out" = "ctdbd" ] ; then
+	echo "GOOD!"
+    elif [ -n "$VALGRIND" -a "$out" = "memcheck" ] ; then
+	# We could check cmdline too if this isn't good enough.
+	echo "GOOD enough!"
+    else
+	echo "BAD!"
+	testfailures=1
+    fi
+    n=$(($n + 1))
+done <<<"$pids_onnode"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/07_ctdb_process_exists.sh
+++ b/ctdb/tests/simple/07_ctdb_process_exists.sh
@ -0,0 +1,78 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb process-exists' shows correct information.
+
+The implementation is creative about how it gets PIDs for existing and
+non-existing processes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. On one of the cluster nodes, get the PID of an existing process
+   (using ps wax).
+3. Run 'ctdb process-exists <pid>' on the node and verify that the
+   correct output is shown.
+4. Run 'ctdb process-exists <pid>' with a pid of a non-existent
+   process and verify that the correct output is shown.
+
+Expected results:
+
+* 'ctdb process-exists' shows the correct output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# Create a background process on $test_node that will last for 60 seconds.
+# It should still be there when we check.
+try_command_on_node $test_node 'sleep 60 >/dev/null 2>&1 & echo $!'
+pid="$out"
+
+echo "Checking for PID $pid on node $test_node"
+# set -e is good, but avoid it here
+status=0
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
+echo "$out"
+
+if [ $status -eq 0 ] ; then
+    echo "OK"
+else
+    echo "BAD"
+    testfailures=1
+fi
+
+# Now just echo the PID of the shell from the onnode process on node
+# 2.  This PID will disappear and PIDs shouldn't roll around fast
+# enough to trick the test...  but there is a chance that will happen!
+try_command_on_node $test_node 'echo $$'
+pid="$out"
+
+echo "Checking for PID $pid on node $test_node"
+# set -e is good, but avoid it here
+status=0
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
+echo "$out"
+
+if [ $status -ne 0 ] ; then
+    echo "OK"
+else
+    echo "BAD"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@ -0,0 +1,58 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb isnotrecmaster'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb isnotrecmaster' on each node.
+
+3. Verify that only 1 node shows the output 'This node is the
+   recmaster' and all the other nodes show the output 'This node is
+   not the recmaster'.
+
+Expected results:
+
+* 'ctdb isnotrecmaster' shows the correct output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+cmd="$CTDB isnotrecmaster || true"
+try_command_on_node all "$cmd"
+echo "Output of \"$cmd\":"
+echo "$out"
+
+num_all_lines=$(echo "$out" |  wc -l)
+num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
+num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
+
+if [ $num_rm_lines -eq 1 ] ; then
+    echo "OK, there is only 1 recmaster"
+else
+    echo "BAD, there are ${num_rm_lines} nodes claiming to be the recmaster"
+    testfailures=1
+fi
+
+if [ $(($num_all_lines - $num_not_rm_lines)) -eq 1 ] ; then
+    echo "OK, all the other nodes claim not to be the recmaster"
+else
+    echo "BAD, there are only ${num_not_rm_lines} nodes claiming not to be the recmaster"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/09_ctdb_ping.sh
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@ -0,0 +1,58 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb ping' command.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb ping' command on one of the nodes and verify that it
+   shows valid and expected output. 
+3. Shutdown one of the cluster nodes, using the 'ctdb shutdown'
+   command. 
+4. Run the 'ctdb ping -n <node>' command from another node to this
+   node. 
+5. Verify that the command is not successful since th ctdb daemon is
+   not running on the node.
+
+Expected results:
+
+* The 'ctdb ping' command shows valid and expected output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB ping -n 1"
+
+sanity_check_output \
+    1 \
+    '^response from 1 time=[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
+    "$out"
+
+try_command_on_node 0 "$CTDB shutdown -n 1"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status 1 disconnected
+
+try_command_on_node -v 0 "! $CTDB ping -n 1"
+
+sanity_check_output \
+    1 \
+    "(: ctdb_control error: 'ctdb_control to disconnected node'|Unable to get ping response from node 1|Node 1 is DISCONNECTED)" \
+    "$out"
+
+echo "Expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@ -0,0 +1,63 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb ip' shows the correct output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb ip' on one of the nodes and verify the list of IP
+   addresses displayed (cross check the result with the output of
+   'ip addr show' on the node).
+3. Verify that colon-separated output is generated with the -Y option.
+
+Expected results:
+
+* 'ctdb ip' shows the list of public IPs being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 1 $CTDB ip -n all
+ips=$(echo "$out" | sed -e '1d')
+colons=$(echo "$ips" | sed -e 's@^@:@' -e 's@$@:@' -e 's@ @:@')
+
+while read ip pnn ; do
+    try_command_on_node $pnn "ip addr show"
+    if [ "${out/inet ${ip}\/}" != "$out" ] ; then
+	echo "GOOD: node $pnn appears to have $ip assigned"
+    else
+	echo "BAD:  node $pnn does not appear to have $ip assigned"
+	testfailures=1
+    fi
+done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
+
+[ "$testfailures" != 1 ] && echo "Looks good!"
+
+cmd="$CTDB -Y ip -n all | sed -e '1d'"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$colons" ] ; then
+    echo "Yep, looks good!"
+else
+    echo "Nope, it looks like this:"
+    echo "$out"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
@ -0,0 +1,84 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb getdebug' works as expected.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the current debug level on a node, using 'ctdb getdebug -n <node>'.
+3. Verify that colon-separated output is generated with the -Y option.
+4. Verify that the '-n all' option shows the debug level on all nodes.
+
+Expected results:
+
+* 'ctdb getdebug' shows the debug level on all the nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+
+try_command_on_node -v 1 "onnode -q all $CTDB getdebug"
+getdebug_onnode="$out"
+
+sanity_check_output \
+    $num_nodes \
+    '^Node [[:digit:]] is at debug level [[:alpha:]]+ \([[:digit:]]\)$' \
+    "$out"
+
+try_command_on_node -v 1 "$CTDB getdebug -n all"
+getdebug_all="$out"
+
+cmd=""
+n=0
+while [ $n -lt $num_nodes ] ; do
+    cmd="${cmd}${cmd:+; }$CTDB getdebug -n $n"
+    n=$(($n + 1))
+done
+try_command_on_node -v 1 "$cmd"
+getdebug_n="$out"
+
+if [ "$getdebug_onnode" = "$getdebug_all" -a \
+    "$getdebug_all" = "$getdebug_n" ] ; then
+    echo "They're the same... cool!"
+else
+    echo "Error: they differ."
+    testfailures=1
+fi
+
+colons=""
+nl="
+"
+while read line ; do
+    t=$(echo "$line" | sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \((-?[[:digit:]])\)$@:\1:\2:@')
+    colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}"
+done <<<"$getdebug_onnode"
+
+cmd="$CTDB -Y getdebug -n all"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$colons" ] ; then
+    echo "Yep, looks good!"
+else
+    echo "Nope, it looks like this:"
+    echo "$out"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/13_ctdb_setdebug.sh
+++ b/ctdb/tests/simple/13_ctdb_setdebug.sh
@ -0,0 +1,85 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb setdebug' works as expected.
+
+This is a little superficial.  It checks that CTDB thinks the debug
+level has been changed but doesn't actually check that logging occurs
+at the new level.
+
+A test should also be added to see if setting the debug value via a
+numerical value works too.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the current debug level on a node, using 'ctdb getdebug'.
+3. Change the debug level to some other value (e.g. EMERG) using
+   'ctdb setdebug'.
+4. Verify that the new debug level is correctly set using 'ctdb getdebug'.
+
+Expected results:
+
+* 'ctdb setdebug' correctly sets the debug level on a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+get_debug ()
+{
+    # Sets; check_debug
+    local node="$1"
+
+    local out
+    
+    try_command_on_node -v $node "$CTDB getdebug"
+    check_debug=$(echo "$out" |
+	sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \(-?[[:digit:]]\)$@\1@')
+}
+
+set_and_check_debug ()
+{
+    local node="$1"
+    local level="$2"
+
+    echo "Setting debug level on node ${node} to ${level}."
+    try_command_on_node $node "$CTDB setdebug ${level}"
+
+    local check_debug
+    get_debug $node
+
+    if [ "$level" = "$check_debug" ] ; then
+	echo "That seemed to work... cool!"
+    else
+	echo "BAD: Debug level should have changed to \"$level\" but it is \"$check_debug\"."
+	testfailures=1
+    fi
+}
+
+get_debug 1
+initial_debug="$check_debug"
+
+new_debug="EMERG"
+[ "$initial_debug" = "$new_debug" ] && new_debug="ALERT"
+
+set_and_check_debug 1 "$new_debug"
+
+if [ "$testfailures" != 1 ] ; then
+    echo "Returning the debug level to its initial value..."
+    set_and_check_debug 1 "$initial_debug"
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/14_ctdb_statistics.sh
+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
@ -0,0 +1,46 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb statistics' works as expected.
+
+This is pretty superficial and could do more validation.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb statistics' on a node, and verify that the output is
+   valid.
+3. Repeat the command with the '-n all' option and verify that the
+   output is valid.
+
+Expected results:
+
+* 'ctdb statistics' shows valid output on all the nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+pattern='^(CTDB version 1|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+[[:alpha:]_]+_latency[[:space:]]+[[:digit:]]+\.[[:digit:]]+[[:space:]]sec)$'
+
+try_command_on_node -v 1 "$CTDB statistics"
+
+sanity_check_output 38 "$pattern" "$out"
+
+try_command_on_node -v 1 "$CTDB statistics -n all"
+
+sanity_check_output 38 "$pattern" "$out"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
@ -0,0 +1,85 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb statisticsreset' works as expected.
+
+This is pretty superficial.  It just checks that a few particular
+items reduce.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb statisticsreset' on all nodes and verify that it executes
+   successfully.
+
+Expected results:
+
+* 'ctdb statisticsreset' executes successfully.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+
+get_stat ()
+{
+    local label="$1"
+    local out="$2"
+
+    echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
+}
+
+check_reduced ()
+{
+    local label="$1"
+    local before="$2"
+    local after="$3"
+
+    if [ $after -lt $before ] ; then
+	echo "GOOD: ${label} reduced from ${before} to ${after}"
+    else
+	echo "BAD: ${label} did not reduce from ${before} to ${after}"
+	testfailures=1
+    fi
+}
+
+n=0
+while [ $n -lt $num_nodes ] ; do
+    echo "Getting initial statistics for node ${n}..."
+    
+    try_command_on_node -v $n $CTDB statistics
+
+    before_req_control=$(get_stat "req_control" "$out")
+    before_reply_control=$(get_stat "reply_control" "$out")
+    before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+
+    try_command_on_node $n $CTDB statisticsreset
+
+    try_command_on_node -v $n $CTDB statistics
+
+    after_req_control=$(get_stat "req_control" "$out")
+    after_reply_control=$(get_stat "reply_control" "$out")
+    after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+
+    check_reduced "req_control" "$before_req_control" "$after_req_control"
+    check_reduced "reply_control" "$before_reply_control" "$after_reply_control"
+    check_reduced "node_packets_recv" "$before_node_packets_recv" "$after_node_packets_recv"
+
+    n=$(($n + 1))
+done
+
+ctdb_test_exit
--- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh
+++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
@ -0,0 +1,118 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that an IP address can be added to a node using 'ctdb addip'.
+
+This test goes to some trouble to figure out which IP address to add
+but assumes a 24-bit subnet mask.  It does not handle IPv6.  It does
+not do any network level checks that the new IP address is reachable
+but simply trusts 'ctdb ip' that the address has been added.  There is
+also an extra prerequisite that the node being added to already has
+public addresses - this is difficult to avoid if the extra address is
+to be sensibly chosen.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+   served.
+3. Add an additional public address to be served by the node, using
+   'ctdb addip'.
+4. Verify that this IP address has been added to the list of IP
+   addresses being served by the node, using the 'ctdb ip' command.
+
+Expected results:
+
+* 'ctdb ip' adds an IP address to the list of public IP addresses
+  being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly.  :-)
+read x test_node <<<"$out"
+
+test_node_ips=""
+all_ips=""
+while read ip pnn ; do
+    all_ips="${all_ips}${all_ips:+ }${ip}"
+    [ "$pnn" = "$test_node" ] && \
+	test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+done <<<"$out"
+
+echo "Selected node ${test_node} with IPs: $test_node_ips"
+
+# Try to find a free IP adddress.  This is inefficient but should
+# succeed quickly.
+try_command_on_node $test_node "ip addr show"
+all_test_node_ips=$(echo "$out" | sed -rn -e 's@^[[:space:]]+inet[[:space:]]+([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+/[[:digit:]]+).*[[:space:]]([^[:space:]]+)+$@\1:\2@p')
+
+add_ip=""
+
+# Use an IP already on one of the nodes, remove the last octet and
+# loop through the possible IP addreses.
+for i in $test_node_ips ; do
+    prefix="${i%.*}"
+    for j in $(seq 1 254) ; do
+	try="${prefix}.${j}"
+	# Try to make sure it isn't used anywhere!
+
+	# First, make sure it isn't an existing public address on the
+	# cluster.
+	for k in $all_ips ; do
+	    [ "$try" = "$k" ] && continue 2
+	done
+
+	# Also make sure it isn't some other address in use on the
+	# node.
+	for k in $all_test_node_ips ; do
+	    [ "$try" = "${k%/*}" ] && continue 2
+	done
+
+	# Get the interface details for $i, which our address is a
+	# close relative of.  This should never fail but it can't hurt
+	# to be careful...
+	for k in $all_test_node_ips ; do
+	    if [ "$i" = "${k%/*}" ] ; then
+			# Found one!
+		add_ip="${try}/${k#*/}"
+		break 3
+	    fi
+	done
+    done
+done
+
+if [ -n "$add_ip" ] ; then
+    echo "Adding IP: ${add_ip/:/ on interface }"
+    try_command_on_node $test_node $CTDB addip ${add_ip/:/ }
+
+    echo "Waiting for IP to be added..."
+    wait_until 60 ips_are_on_nodeglob $test_node $test_node_ips ${add_ip%/*}
+
+    echo "That worked!"
+else
+    echo "BAD: Unable to find IP address to add."
+    testfailures=1
+fi
+
+echo "Restarting cluster to restore configuration..."
+restart_ctdb
+
+ctdb_test_exit
--- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
@ -0,0 +1,75 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that a node's public IP address can be deleted using 'ctdb deleteip'.
+
+This test does not do any network level checks that the IP address is
+no longer reachable but simply trusts 'ctdb ip' that the address has
+been deleted.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+   served.
+3. Delete one public IP address being be served by the node, using
+   'ctdb delip'.
+4. Verify that the delete IP address is no longer listed using the
+   'ctdb ip' command.
+
+Expected results:
+
+* 'ctdb delip' removes an IP address from the list of public IP
+  addresses being served by a node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
+
+# Select an IP/node to remove.
+num_ips=$(echo "$out" | wc -l)
+num_to_remove=$(($RANDOM % $num_ips))
+
+# Find the details in the list.
+i=0
+while [ $i -le $num_to_remove ] ; do
+    read ip_to_remove test_node
+    i=$(($i + 1))
+done <<<"$out"
+
+echo "Attempting to remove ${ip_to_remove} from node ${test_node}."
+try_command_on_node $test_node $CTDB delip $ip_to_remove
+
+echo "Sleeping..."
+sleep_for 1
+
+test_node_ips=""
+while read ip pnn ; do
+    [ "$pnn" = "$test_node" ] && \
+	test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
+    echo "That worked!  Restarting cluster to restore configuration..."
+    restart_ctdb
+else
+    echo "BAD: The remove IP address is still there!"
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/18_ctdb_freeze.sh
+++ b/ctdb/tests/simple/18_ctdb_freeze.sh
@ -0,0 +1,51 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify 'ctdb freeze' works correctly.
+
+This is a superficial test that simply checks that 'ctdb statistics'
+reports the node becomes frozen.  No checks are done to ensure that
+client access to databases is blocked.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb freeze -n <node>' to freeze the databases on one of the
+   nodes.
+3. Run 'ctdb statistics' to verify that 'frozen' has the value '1' on
+   the node.
+
+Expected results:
+
+* When the database is frozen, the 'frozen' variable in the
+  'ctdb statistics' output is set to 1 on the node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+echo "Freezing node $test_node"
+
+try_command_on_node 0 $CTDB freeze -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
+
+echo "That worked!  Restarting cluster to restore configuration..."
+
+restart_ctdb
+
+ctdb_test_exit
--- a/ctdb/tests/simple/19_ctdb_thaw.sh
+++ b/ctdb/tests/simple/19_ctdb_thaw.sh
@ -0,0 +1,57 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify 'ctdb thaw' works correctly.
+
+This is a superficial test that simply checks that 'ctdb statistics'
+reports the node becomes unfrozen.  No checks are done to ensure that
+client access to databases is unblocked.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb freeze -n <node>' to freeze the databases on one of the
+   nodes.
+3. Run 'ctdb statistics' to verify that 'frozen' has the value '1' on
+   the node.
+4, Now run 'ctdb thaw -n <node>' on the same node.
+5. Run 'ctdb statistics' to verify that 'frozen' once again has the
+   value '0' on the node.
+
+
+Expected results:
+
+* 'ctdb thaw' causes a node to 'thaw' and the status change can be
+  seem via 'ctdb statistics'.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+echo "Freezing node $test_node"
+
+try_command_on_node 0 $CTDB freeze -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
+
+echo "That worked!  Now thawing node $test_node"
+
+try_command_on_node 0 $CTDB thaw -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unfrozen
+
+ctdb_test_exit
--- a/ctdb/tests/simple/20_ctdb_getmonmode.sh
+++ b/ctdb/tests/simple/20_ctdb_getmonmode.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify 'ctdb getmonmode' works correctly.
+
+This test doesn't actually verify that enabling and disabling
+monitoring mode actually does that.  It trusts ctdb that the
+monitoring mode is modified as requested.  21_ctdb_disablemonitor.sh
+does some more useful checking.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb getmodmode -n <node>' to get the current monitoring mode.
+3. Verify that it shows monitoring as 'active'.
+4. Verify that the command prints the output in colon-separated format
+   when run with the '-Y' option.
+5. Disable monitoring on the node using 'ctdb disablemonitor'.
+6. Verify that it shows monitoring as 'disabled'.
+
+Expected results:
+
+* 'ctdb getmonmode' works as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+try_command_on_node -v 0 $CTDB getmonmode -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:ACTIVE \(0\)$' \
+    "$out"
+
+colons=$(printf ':mode:\n:0:')
+
+try_command_on_node -v 0 $CTDB -Y getmonmode -n $test_node
+
+if [ "$out" = "$colons" ] ; then
+    echo "Looks OK"
+else
+    echo "BAD: -Y output isn't what was expected"
+    testfailures=1
+fi
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+echo "That worked!  Restarting cluster to restore configuration..."
+
+restart_ctdb
+
+ctdb_test_exit
--- a/ctdb/tests/simple/21_ctdb_disablemonitor.sh
+++ b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
@ -0,0 +1,101 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb disablemonitor' works correctly.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+* 00_ctdb_install_eventscript.sh successfully installed its event
+  script.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Create a file called /tmp/ctdb-test-unhealthy-trigger.<node> on a
+   node and verify that the status of the node changes to unhealthy
+   within the interval indicated by the MonitorInterval variable.
+3. Check that the file /tmp/ctdb-test-unhealthy-detected.<node> is
+   created, indicating that the event script is the reason the node
+   has been marked as unhealthy.
+4. Now disable monitoring on the node using 'ctdb disablemonitor -n <node>.
+5. Verify that the message 'Monitoring mode:DISABLED' is printed.
+6. Remove /tmp/ctdb-test-unhealthy-detected.<node> and ensure that it
+   is not recreated within the interval indicated by the
+   MonitorInterval variable.
+7. Remove /tmp/ctdb-test-unhealthy-trigger.<node>.
+8. Verify that the status of the node continues to be 'UNHEALTHY',
+   since monitoring has been disabled.
+
+Expected results:
+
+* When monitoring is disabled, event scripts are not executed and the
+  state of nodes is not monitored.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
+monitor_interval="${out#*= }"
+echo "Monitor interval on node $test_node is $monitor_interval seconds."
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${test_node}"
+detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
+
+recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
+try_command_on_node $test_node touch "$recovered_flag"
+
+ctdb_test_exit_hook="onnode $test_node rm -vf $trigger"
+
+echo "Creating trigger file on node $test_node to make it unhealthy..."
+try_command_on_node $test_node touch "$trigger"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
+
+try_command_on_node -v $test_node ls -l "$detected"
+
+# Wait until recovery is complete before disabling monitoring,
+# otherwise completion of the recover can turn monitoring back on!
+echo "Waiting until recovery is complete..."
+wait_until 30 onnode $test_node ! test -e "$recovered_flag"
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:DISABLED$' \
+    "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+try_command_on_node -v $test_node rm -v "$detected"
+
+sleep_for $monitor_interval
+
+try_command_on_node $test_node test ! -e "$detected"
+
+echo "OK: flag file was not recreated so monitoring must be disabled."
+
+echo "Removing trigger file.  Monitoring is disabled so node will stay unhealthy."
+
+try_command_on_node -v $test_node rm -v "$trigger"
+
+sleep_for $monitor_interval
+
+onnode 0 $CTDB_TEST_WRAPPER node_has_status $test_node unhealthy
+
+echo "OK, that all worked.  Expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/22_ctdb_enablemonitor.sh
+++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
@ -0,0 +1,101 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb enablemonitor' works correctly.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+* 00_ctdb_install_eventscript.sh successfully installed its event
+  script.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable monitoring on a node using 'ctdb disablemonitor -n <node>.
+3. Create a file called /tmp/ctdb-test-unhealthy-trigger.<node> on the
+   node.
+4. Verify that the status of the node does not change to unhealthy
+   within the interval indicated by the MonitorInterval variable,
+   since monitoring is disabled.
+5. Now enable monitoring on the node using 'ctdb enablemonitor -n <node>.
+6. Verify that the status of the node changes to unhealthy within the
+   interval indicated by the MonitorInterval variable.
+7. Check that the file /tmp/ctdb-test-unhealthy-detected.<node> is
+   created, indicating that the event script is the reason the node
+   has been marked as unhealthy.
+
+Expected results:
+
+* When monitoring is enabled on a node, event scripts are executed and
+  status changes are monitored.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
+monitor_interval="${out#*= }"
+echo "Monitor interval on node $test_node is $monitor_interval seconds."
+
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:DISABLED$' \
+    "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${test_node}"
+detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
+
+recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
+try_command_on_node $test_node touch "$recovered_flag"
+
+ctdb_test_exit_hook="onnode $test_node rm -vf $trigger; restart_ctdb"
+
+echo "Creating trigger file on node $test_node to see if it goes unhealthy..."
+try_command_on_node $test_node touch "$trigger"
+
+sleep_for $monitor_interval
+
+try_command_on_node 0 $CTDB_TEST_WRAPPER node_has_status $test_node healthy
+
+try_command_on_node $test_node test ! -e "$detected"
+
+echo "OK: flag file was not created so monitoring must be disabled."
+
+try_command_on_node -v 0 $CTDB enablemonitor -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:ACTIVE$' \
+    "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monon
+
+sleep_for $monitor_interval
+
+try_command_on_node $test_node test -e "$detected"
+
+echo "OK: flag file was created so monitoring must be enabled."
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
+
+try_command_on_node -v $test_node ls -l "$detected"
+
+echo "OK, that all worked.  Expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@ -0,0 +1,105 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb moveip' allows movement of public IPs between cluster nodes.
+
+To work, this test unsets DeterministicIPs and sets NoIPFailback.
+
+This test does not do any network level checks that the IP address is
+no longer reachable but simply trusts 'ctdb ip' that the address has
+been deleted.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+   served.
+3. Use 'ctdb moveip' to move an address from one node to another.
+4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node.
+
+Expected results:
+
+* 'ctdb moveip' allows an IP address to be moved between cluster nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+if [ $num_nodes -lt 2 ] ; then
+    echo "Less than 2 nodes!"
+    exit 1
+fi
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
+
+sanity_check_ips "$out"
+
+# Select an IP/node to move.
+num_ips=$(echo "$out" | wc -l)
+num_to_move=$(($RANDOM % $num_ips))
+
+# Find the details in the list.
+i=0
+while [ $i -le $num_to_move ] ; do
+    read ip_to_move test_node
+    i=$(($i + 1))
+done <<<"$out"
+
+# Can only move address to a node that is willing to host $ip_to_move.
+# This inefficient but shouldn't take long or get stuck.
+to_node=$test_node
+while [ $test_node -eq $to_node ] ; do
+    n=$(($RANDOM % $num_ips))
+    i=0
+    while [ $i -le $n ] ; do
+	read x to_node
+	i=$(($i + 1))
+    done <<<"$out"
+done
+
+echo "Turning off DeterministicIPs..."
+try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all
+
+echo "Turning on NoIPFailback..."
+try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all
+
+echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}."
+try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ip_to_move ; then
+    echo "IP moved from ${test_node}."
+else
+    echo "BAD: IP didn't move from ${test_node}."
+    exit 1
+fi
+
+if wait_until_ips_are_on_nodeglob "$to_node" $ip_to_move ; then
+    echo "IP moved to ${to_node}."
+else
+    echo "BAD: IP didn't move to ${to_node}."
+    exit 1
+fi
+
+echo "OK, that worked... expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@ -0,0 +1,75 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb getdbmap' operates as expected.
+
+This test creates some test databases using 'ctdb attach'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the database on using 'ctdb getdbmap'.
+3. Verify that the output is valid.
+
+Expected results:
+
+* 'ctdb getdbmap' shows a valid listing of databases.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+make_temp_db_filename ()
+{
+    dd if=/dev/urandom count=1 bs=512 2>/dev/null |
+    md5sum |
+    awk '{printf "%s.tdb\n", $1}'
+}
+
+try_command_on_node -v 0 "$CTDB getdbmap"
+
+db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
+
+sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+
+num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+
+for i in $(seq 1 5) ; do
+    f=$(make_temp_db_filename)
+    echo "Creating test database: $f"
+    try_command_on_node 0 $CTDB attach "$f"
+    try_command_on_node 0 $CTDB getdbmap
+    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+    num=$(echo "$out" | sed -n -e '1s/^.*://p')
+    if [ $num = $(($num_db_init + $i)) ] ; then
+	echo "OK: correct number of additional databases"
+    else
+	echo "BAD: no additional database"
+	exit 1
+    fi
+    if [ "${out/name:${f} /}" != "$out" ] ; then
+	echo "OK: getdbmap knows about \"$f\""
+    else
+	echo "BAD: getdbmap does not know about \"$f\""
+	exit 1
+    fi
+done
+
+echo "OK, that worked... expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/25_dumpmemory.sh
+++ b/ctdb/tests/simple/25_dumpmemory.sh
@ -0,0 +1,54 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb dumpmemory' shows expected output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb dumpmemory' and verify that it shows expected output
+3. Verify that the command takes the '-n all' option and that it
+   causes output for all nodes to be displayed.
+
+Expected results:
+
+* 'ctdb dumpmemory' sows valid output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node -v 0 "$CTDB dumpmemory"
+
+pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$'
+
+sanity_check_output 10 "$pat" "$out"
+
+echo "Checking output using '-n all'..."
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+try_command_on_node 0 "$CTDB dumpmemory" -n all
+sanity_check_output 10 "$pat" "$out"
+
+if [ $(fgrep -c 'full talloc report on' <<<"$out") -eq  $num_nodes ] ; then
+    echo "OK: there looks to be output for all $num_nodes nodes"
+else
+    echo "BAD: there not look to be output for all $num_nodes nodes"
+    exit 1
+fi    
+
+ctdb_test_exit
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@ -0,0 +1,71 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify an error occurs if a ctdb command is run against a node without a ctdbd.
+
+That is, check that an error message is printed if an attempt is made
+to execute a ctdb command against a node that is not running ctdbd.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Shutdown ctdb on a node using 'ctdb shutdown -n <node>'.
+3. Verify that the status of the node changes to 'DISCONNECTED'.
+4. Now run 'ctdb ip -n <node>' from another node.
+5. Verify that an error message is printed stating that the node is
+   disconnected.
+6. Execute some other commands against the shutdown node.  For example,
+   disable, enable, ban, unban, listvars.
+7. For each command, verify that an error message is printed stating
+   that the node is disconnected. 
+
+Expected results:
+
+* For a node on which ctdb is not running, all commands display an
+  error message stating that the node is disconnected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+echo "There are $num_nodes nodes."
+
+echo "Shutting down node ${test_node}..."
+try_command_on_node $test_node $CTDB shutdown
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disconnected
+
+pat="ctdb_control error: 'ctdb_control to disconnected node'|Node $test_node is DISCONNECTED"
+
+for i in ip disable enable "ban 0" unban listvars ; do
+    try_command_on_node -v 0 ! $CTDB $i -n $test_node
+
+    if egrep -q "$pat" <<<"$out" ; then
+	echo "OK: \"ctdb ${i}\" fails with \"disconnected node\""
+    else
+	echo "BAD: \"ctdb ${i}\" does not fail with \"disconnected node\""
+	exit 1
+    fi
+done
+
+echo "That all looks OK.  Restarting cluster..."
+
+restart_ctdb
+
+ctdb_test_exit
--- a/ctdb/tests/simple/31_ctdb_disable.sh
+++ b/ctdb/tests/simple/31_ctdb_disable.sh
@ -0,0 +1,72 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb disable'.
+
+This is a superficial test of the 'ctdb disable' command.  It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly.  Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the IP addreses served by the disabled node are failed
+   over to other nodes.
+
+Expected results:
+
+* The status of the disabled node changes as expected and IP addresses
+  failover as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly.  :-)
+read x test_node <<<"$out"
+
+ips=""
+while read ip pnn ; do
+    if [ "$pnn" = "$test_node" ] ; then
+	ips="${ips}${ips:+ }${ip}"
+    fi
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+echo "Disabling node $test_node"
+
+try_command_on_node 1 $CTDB disable -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "Expect a restart..."
+
+ctdb_test_exit
--- a/ctdb/tests/simple/32_ctdb_enable.sh
+++ b/ctdb/tests/simple/32_ctdb_enable.sh
@ -0,0 +1,91 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb enable'.
+
+This is a superficial test of the 'ctdb enable' command.  It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly.  Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the public IP addreses served by the disabled node are
+   failed over to other nodes.
+5. Enable the disabled node using 'ctdb enable -n '<node>'.
+6. Verify that the status changes back to 'OK'.
+7. Verify that the public IP addreses served by the disabled node are
+   failed back to the node.
+
+
+Expected results:
+
+* The status of a re-enabled node changes as expected and IP addresses
+  fail back as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+########################################
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we just want a node that has public IPs.
+# This will work and is economically semi-randomly.  :-)
+read x test_node <<<"$out"
+
+ips=""
+while read ip pnn ; do
+    if [ "$pnn" = "$test_node" ] ; then
+	ips="${ips}${ips:+ }${ip}"
+    fi
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "Reenabling node $test_node"
+try_command_on_node 1 $CTDB enable -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node enabled
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+#      NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "All done!"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/41_ctdb_ban.sh
+++ b/ctdb/tests/simple/41_ctdb_ban.sh
@ -0,0 +1,100 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb ban' command.
+
+This is a superficial test of the 'ctdb ban' command.  It trusts
+information from CTDB that indicates that the IP failover has
+happened correctly.  Another test should check that the failover
+has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
+3. Before the ban timeout expires, verify that the status of the
+   node changes to 'banned'.
+4. Verify that the public IP addresses that were being served by
+   the node are failed over to one of the other nodes.
+5. When the ban expires ensure that the status of the node changes
+   back to 'OK' and that the public IP addresses move back to the
+   node.
+
+Expected results:
+
+* The status of the banned nodes changes as expected and IP addresses
+  failover as expected.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# When selecting test_node we want a node that has public IPs and that
+# is not the recmaster.  We pick the first one that satisfies both
+# conditions.  We avoid the recmaster because banning the recmaster
+# (obviously) causes the recmaster to change... and changing the
+# recmaster causes all nodes to become unbanned!
+test_node=""
+
+ips=""
+while read ip pnn ; do
+    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ -z "$test_node" ] ; then
+    echo "BAD: unable to select a suitable node for banning."
+    exit 1
+fi
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+ban_time=15
+
+echo "Banning node $test_node for $ban_time seconds"
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "Sleeping until ban expires..."
+sleep_for $ban_time
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+#      NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/42_ctdb_unban.sh
+++ b/ctdb/tests/simple/42_ctdb_unban.sh
@ -0,0 +1,97 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb unban' command.
+
+This is a superficial test of the 'ctdb uban' command.  It trusts
+information from CTDB that indicates that the IP failover and failback
+has happened correctly.  Another test should check that the failover
+and failback has actually happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Ban one of the nodes using the 'ctdb ban <timeout>' command.
+3. Before the ban timeout expires, verify that the status of the
+   node changes to 'banned'.
+4. Verify that the public IP addresses that were being served by
+   the node are failed over to one of the other nodes.
+5. Before the ban timeout expires, use 'ctdb unban' to unban the
+   node.
+6. Verify that the status of the node changes back to 'OK' and that
+   the public IP addresses move back to the node.
+
+Expected results:
+
+* The 'ctdb unban' command successfully unbans a banned node.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
+echo "Getting list of public IPs..."
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
+
+# See 41_ctdb_ban.sh for an explanation of why test_node is chosen
+# like this.
+test_node=""
+
+ips=""
+while read ip pnn ; do
+    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
+done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+
+if [ -z "$test_node" ] ; then
+    echo "BAD: unable to select a suitable node for banning."
+    exit 1
+fi
+
+echo "Selected node ${test_node} with IPs: $ips"
+
+ban_time=60
+
+echo "Banning node $test_node for $ban_time seconds"
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
+
+# Avoid a potential race condition...
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+echo "Unbanning node $test_node"
+try_command_on_node 1 $CTDB unban -n $test_node
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
+
+# BUG: this is only guaranteed if DeterministicIPs is 1 and
+#      NoIPFailback is 0.
+if wait_until_ips_are_on_nodeglob "$test_node" $ips ; then
+    echo "All IPs moved."
+else
+    echo "Some IPs didn't move."
+    testfailures=1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/51_ctdb_bench.sh
+++ b/ctdb/tests/simple/51_ctdb_bench.sh
@ -0,0 +1,94 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Run the ctdb_bench test and sanity check the output.
+
+This doesn't test for performance regressions or similarly anything
+useful.  Only vague sanity checking of results is done.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run ctdb_bench on all nodes with default options.
+3. Ensure that the number of +ve and -ive messages are within 1% of
+   each other.
+4. Ensure that the number of messages per second is greater than 10.
+
+Expected results:
+
+* ctdb_bench runs without error and prints reasonable results.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+echo "Running ctdb_bench on all $num_nodes nodes."
+try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_bench -n $num_nodes
+
+# Get the last line of output.
+while read line ; do
+    prev=$line
+done <<<"$out"
+
+pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$'
+sanity_check_output 1 "$pat" "$out"
+
+# $prev should look like this:
+#    Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373)
+stuff="${prev##*Ring: }"
+mps="${stuff% msgs/sec*}"
+
+if [ ${mps%.*} -ge 10 ] ; then
+    echo "OK: $mps msgs/sec >= 10 msgs/sec"
+else
+    echo "BAD: $mps msgs/sec < 10 msgs/sec"
+    exit 1
+fi
+
+stuff="${stuff#*msgs/sec (+ve=}"
+positive="${stuff%% *}"
+
+if [ $positive -gt 0 ] ; then
+    echo "OK: +ive ($positive) > 0"
+else
+    echo "BAD: +ive ($positive) = 0"
+    exit 1
+fi
+
+stuff="${stuff#*-ve=}"
+negative="${stuff%)}"
+
+if [ $negative -gt 0 ] ; then
+    echo "OK: -ive ($negative) > 0"
+else
+    echo "BAD: -ive ($negative) = 0"
+    exit 1
+fi
+
+perc_diff=$(( ($positive - $negative) * 100 / $positive ))
+perc_diff=${perc_diff#-}
+
+if [ $perc_diff -le 1 ] ; then
+    echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= 1%"
+else
+    echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
+    exit 1
+fi
+
+
+ctdb_test_exit
--- a/ctdb/tests/simple/52_ctdb_fetch.sh
+++ b/ctdb/tests/simple/52_ctdb_fetch.sh
@ -0,0 +1,93 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Run the ctdb_fetch test and sanity check the output.
+
+This doesn't test for performance regressions or similarly anything
+useful.  Only vague sanity checking of results is done.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run ctdb_fetch on all nodes with default options.
+3. Ensure that the number of +ve and -ive messages are within 1% of
+   each other.
+4. Ensure that the number of messages per second is greater than 10.
+
+Expected results:
+
+* ctdb_fetch runs without error and prints reasonable results.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+echo "Running ctdb_fetch on all $num_nodes nodes."
+try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_fetch -n $num_nodes
+
+# Get the last line of output.
+while read line ; do
+    prev=$line
+done <<<"$out"
+
+pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$'
+sanity_check_output 1 "$pat" "$out"
+
+# $prev should look like this:
+#    Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373)
+stuff="${prev##*Ring: }"
+mps="${stuff% msgs/sec*}"
+
+if [ ${mps%.*} -ge 10 ] ; then
+    echo "OK: $mps msgs/sec >= 10 msgs/sec"
+else
+    echo "BAD: $mps msgs/sec < 10 msgs/sec"
+    exit 1
+fi
+
+stuff="${stuff#*msgs/sec (+ve=}"
+positive="${stuff%% *}"
+
+if [ $positive -gt 0 ] ; then
+    echo "OK: +ive ($positive) > 0"
+else
+    echo "BAD: +ive ($positive) = 0"
+    exit 1
+fi
+
+stuff="${stuff#*-ve=}"
+negative="${stuff%)}"
+
+if [ $negative -gt 0 ] ; then
+    echo "OK: -ive ($negative) > 0"
+else
+    echo "BAD: -ive ($negative) = 0"
+    exit 1
+fi
+
+perc_diff=$(( ($positive - $negative) * 100 / $positive ))
+perc_diff=${perc_diff#-}
+
+if [ $perc_diff -le 1 ] ; then
+    echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= 1%"
+else
+    echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
+    exit 1
+fi
+
+ctdb_test_exit
--- a/ctdb/tests/simple/53_ctdb_transaction.sh
+++ b/ctdb/tests/simple/53_ctdb_transaction.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_transaction test succeeds.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_transaction on each node with a 30 second
+   timeout.
+3. Ensure that all ctdb_transaction processes complete successfully.
+
+Expected results:
+
+* ctdb_transaction runs without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=30"
+
+echo "Running ctdb_transaction on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/61_ctdb_persistent_safe.sh
+++ b/ctdb/tests/simple/61_ctdb_persistent_safe.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_persistent test succeeds for safe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+   timeout.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests safe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --timelimit=30"
+
+echo "Running ctdb_persistent on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
+++ b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_persistent test succeeds for unsafe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+   timeout and with the --unsafe-writes option.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests unsafe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_persistent --unsafe-writes --timelimit=30"
+
+echo "Running ctdb_persistent --unsafe-writes on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
--- a/ctdb/tests/simple/99_ctdb_uninstall_eventscript.sh
+++ b/ctdb/tests/simple/99_ctdb_uninstall_eventscript.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Uninstall the event script used for testing..
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. 
+
+Expected results:
+
+* The script is successfully uninstalled from all nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+uninstall_eventscript "00.ctdb_test_trigger"
--- a/ctdb/tests/src/ctdb_bench.c
+++ b/ctdb/tests/src/ctdb_bench.c
@ -22,6 +22,8 @@
 #include "system/filesys.h"
 #include "popt.h"
 #include "cmdline.h"
+#include "ctdb.h"
+#include "ctdb_private.h"

 #include <sys/time.h>
 #include <time.h>
@ -89,8 +91,9 @@ static void ring_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
 	int incr = *(int *)data.dptr;
 	int *count = (int *)private_data;
 	int dest;
+
 	(*count)++;
-	dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
+	dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes;
 	ctdb_send_message(ctdb, dest, srvid, data);
 	if (incr == 1) {
 		msg_plus++;
@ -99,6 +102,50 @@ static void ring_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
 	}
 }

+
+void send_start_messages(struct ctdb_context *ctdb, int incr)
+{
+	/* two messages are injected into the ring, moving
+	   in opposite directions */
+	int dest;
+	TDB_DATA data;
+		
+	data.dptr = (uint8_t *)&incr;
+	data.dsize = sizeof(incr);
+
+	dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes;
+	ctdb_send_message(ctdb, dest, 0, data);
+}
+
+static void each_second(struct event_context *ev, struct timed_event *te, 
+					 struct timeval t, void *private_data)
+{
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+
+	/* we kickstart the ring into action by inserting messages from node
+	   with pnn 0.
+	   it may happen that some other node does not yet have ctdb_bench
+	   running in which case the ring is broken and the messages are lost.
+	   if so, once every second try again to restart the ring
+	*/
+	if (msg_plus == 0) {
+//		printf("no messages recevied, try again to kickstart the ring in forward direction...\n");
+		send_start_messages(ctdb, 1);
+	}
+	if (msg_minus == 0) {
+//		printf("no messages recevied, try again to kickstart the ring in reverse direction...\n");
+		send_start_messages(ctdb, -1);
+	}
+	event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb);
+}
+
+static void dummy_event(struct event_context *ev, struct timed_event *te, 
+					 struct timeval t, void *private_data)
+{
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+	event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb);
+}
+
 /*
  benchmark sending messages in a ring around the nodes
 */
@ -107,25 +154,12 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	int pnn=ctdb_get_pnn(ctdb);

 	if (pnn == 0) {
-		/* two messages are injected into the ring, moving
-		   in opposite directions */
-		int dest, incr;
-		TDB_DATA data;
-		
-		data.dptr = (uint8_t *)&incr;
-		data.dsize = sizeof(incr);
-
-		incr = 1;
-		dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
-		ctdb_send_message(ctdb, dest, 0, data);
-		
-		incr = -1;
-		dest = (ctdb_get_pnn(ctdb) + incr) % num_nodes;
-		ctdb_send_message(ctdb, dest, 0, data);
+		event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb);
+	} else {
+		event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb);
 	}
-	
-	start_timer();

+	start_timer();
 	while (end_timer() < timelimit) {
 		if (pnn == 0 && msg_count % 10000 == 0) {
 			printf("Ring: %.2f msgs/sec (+ve=%d -ve=%d)\r", 
@ -139,17 +173,6 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	       msg_count/end_timer(), msg_plus, msg_minus);
 }

-/*
-  handler for reconfigure message
-*/
-static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, 
-				TDB_DATA data, void *private_data)
-{
-	int *ready = (int *)private_data;
-	*ready = 1;
-}
-
-
 /*
  main program
 */
@ -172,7 +195,6 @@ int main(int argc, const char *argv[])
 	int ret;
 	poptContext pc;
 	struct event_context *ev;
-	int cluster_ready=0;

 	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);

@ -197,9 +219,6 @@ int main(int argc, const char *argv[])
 	/* initialise ctdb */
 	ctdb = ctdb_cmdline_client(ev);

-	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, 
-				 &cluster_ready);
-
 	/* attach to a specific database */
 	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
 	if (!ctdb_db) {
--- a/ctdb/tests/src/ctdb_fetch.c
+++ b/ctdb/tests/src/ctdb_fetch.c
--- a/ctdb/tests/src/ctdb_persistent.c
+++ b/ctdb/tests/src/ctdb_persistent.c
--- a/ctdb/tests/src/ctdb_randrec.c
+++ b/ctdb/tests/src/ctdb_randrec.c
--- a/ctdb/tests/src/ctdb_store.c
+++ b/ctdb/tests/src/ctdb_store.c
--- a/ctdb/tests/src/ctdb_transaction.c
+++ b/ctdb/tests/src/ctdb_transaction.c
--- a/ctdb/tests/src/ctdb_traverse.c
+++ b/ctdb/tests/src/ctdb_traverse.c
--- a/ctdb/tests/src/rb_perftest.c
+++ b/ctdb/tests/src/rb_perftest.c
--- a/ctdb/tests/src/rb_test.c
+++ b/ctdb/tests/src/rb_test.c
--- a/ctdb/tests/start_daemons.sh
+++ b/ctdb/tests/start_daemons.sh
@ -1,42 +0,0 @@
-#!/bin/sh
-
-NUMNODES=2
-if [ $# -gt 0 ]; then
-    NUMNODES=$1
-fi
-shift
-
-NODES="./tests/nodes.txt"
-rm -f $NODES
-for i in `seq 1 $NUMNODES`; do
-  if [ "${CTDB_USE_IPV6}x" != "x" ]; then
-    echo ::$i >> $NODES
-    ip addr add ::$i/128 dev lo
-  else
-    echo 127.0.0.$i >> $NODES
-  fi
-done
-
-killall -q ctdbd
-rm -rf test.db/persistent/*
- 
-CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --event-script-dir=tests/events.d --logfile=- -d 0 --dbdir=test.db --dbdir-persistent=test.db/persistent $*"
-
-echo "Starting $NUMNODES ctdb daemons"
-for i in `seq 1 $NUMNODES`; do
-    if [ `id -u` -eq 0 ]; then
-        CTDB_OPTIONS="$CTDB_OPTIONS --public-interface=lo"
-    fi
-
-    $VALGRIND bin/ctdbd --socket=sock.$i $CTDB_OPTIONS || exit 1
-done
-ln -sf $PWD/sock.1 /tmp/ctdb.socket || exit 1
-
-while bin/ctdb status | egrep "DISCONNECTED|UNHEALTHY" > /dev/null; do
-    echo "`date` Waiting for recovery"
-    sleep 1;
-done
-
-echo "$NUMNODES daemons started"
-
-exit 0
--- a/ctdb/tests/transaction.sh
+++ b/ctdb/tests/transaction.sh
@ -1,28 +0,0 @@
-#!/bin/sh
-
-NUMNODES=4
-if [ $# -gt 0 ]; then
-    NUMNODES=$1
-fi
-
-killall -9 -q ctdb_transaction ctdbd
-
-rm -rf test.db/transaction
-
-echo "Starting $NUMNODES daemons for transaction writes"
-tests/start_daemons.sh $NUMNODES || exit 1
-
-trap 'echo "Killing test"; killall -9 -q ctdbd ctdb_transaction; exit 1' INT TERM
-
-VALGRIND="valgrind -q"
-for i in `seq 1 $NUMNODES`; do
-  $VALGRIND bin/ctdb_transaction --timelimit 30 --socket sock.$i $* &
-  $VALGRIND bin/ctdb_transaction --timelimit 30 --socket sock.$i $* &
-done
-wait
-
-echo "Shutting down"
-bin/ctdb shutdown -n all --socket=sock.1
-killall -9 ctdbd
-
-exit 0
--- a/ctdb/tools/onnode
+++ b/ctdb/tools/onnode
@ -148,6 +148,7 @@ get_nodes_with_status ()
    esac

    if [ -z "$ctdb_status_output" ] ; then
+	# FIXME: need to do something if $CTDB_NODES_SOCKETS is set.
 	ctdb_status_output=$(ctdb -Y status 2>/dev/null)
 	if [ $? -ne 0 ] ; then
 	    echo "${prog}: unable to get status of CTDB nodes" >&2
@ -177,8 +178,14 @@ get_nodes_with_status ()
 ctdb_recmaster=""
 get_nodes ()
 {
-    [ -f "$CTDB_NODES_FILE" ] || CTDB_NODES_FILE=/etc/ctdb/nodes
-    local all_nodes=$(egrep '^[[:alnum:]]' $CTDB_NODES_FILE)
+    local all_nodes
+
+    if [ -n "$CTDB_NODES_SOCKETS" ] ; then 
+	all_nodes="$CTDB_NODES_SOCKETS"
+    else
+	[ -f "$CTDB_NODES_FILE" ] || CTDB_NODES_FILE=/etc/ctdb/nodes
+	all_nodes=$(egrep '^[[:alnum:]]' $CTDB_NODES_FILE)
+    fi

    local nodes=""
    local n
@ -210,20 +217,29 @@ get_nodes ()
    done
 }

+fakessh ()
+{
+    CTDB_SOCKET="$1" sh -c "$2"
+}
+
 ######################################################################

 parse_options "$@"

 $current && command="cd $PWD && $command"

-SSH_OPTS=
-# Could "2>/dev/null || true" but want to see errors from typos in file.
-[ -r /etc/ctdb/onnode.conf ] && . /etc/ctdb/onnode.conf
-[ -n "$SSH" ] || SSH=ssh
-if [ "$SSH" = "ssh" ] ; then
-    ssh_opts="-n"
-else
-    : # rsh? All bets are off!
+ssh_opts=
+if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+    SSH=fakessh
+else 
+    # Could "2>/dev/null || true" but want to see errors from typos in file.
+    [ -r /etc/ctdb/onnode.conf ] && . /etc/ctdb/onnode.conf
+    [ -n "$SSH" ] || SSH=ssh
+    if [ "$SSH" = "ssh" ] ; then
+	ssh_opts="-n"
+    else
+	: # rsh? All bets are off!
+    fi
 fi

 ######################################################################