From 625e0c5aea92de751a806b5d27af49d2441d1135 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 11:21:33 +1000
Subject: [PATCH 01/29] Tests - add -T (trace) option to eventscripts
 run_test.sh

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 08eeb788523afe19760b00f40339b702eb193712)
---
 ctdb/tests/eventscripts/run_tests.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ctdb/tests/eventscripts/run_tests.sh b/ctdb/tests/eventscripts/run_tests.sh
index cb2c8f29507..ec17b5f7a71 100755
--- a/ctdb/tests/eventscripts/run_tests.sh
+++ b/ctdb/tests/eventscripts/run_tests.sh
@@ -15,6 +15,12 @@ for i ; do
 	    export EVENTSCRIPT_TESTS_VERBOSE="yes"
 	    shift
 	    ;;
+	-T)
+	    # This will cause tests to fail but is good for debugging
+	    # individual tests when they fail.
+	    export EVENTSCRIPTS_TESTS_TRACE="sh -x"
+	    shift
+	    ;;
 	-*)
 	    opts="$opts $i"
 	    shift

From f3c590c199721117869d19294d7065493b3613ca Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 11:22:51 +1000
Subject: [PATCH 02/29] Tests - eventscripts exportfs stub should splits lines

The real exportfs splits lines longer than 15 characters.  The stub
should do that too...

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit fb3e123b097d9e36d281c3ab4f3e9a4799fa27bc)
---
 ctdb/tests/eventscripts/stubs/exportfs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ctdb/tests/eventscripts/stubs/exportfs b/ctdb/tests/eventscripts/stubs/exportfs
index 72a11d57413..46c65225be1 100755
--- a/ctdb/tests/eventscripts/stubs/exportfs
+++ b/ctdb/tests/eventscripts/stubs/exportfs
@@ -3,5 +3,11 @@
 opts="10.0.0.0/16(rw,async,insecure,no_root_squash,no_subtree_check)"
 
 for i in $FAKE_SHARES ; do
-    echo "${i} ${opts}"
+    # Directories longer than 15 characters are printed on their own
+    # line.
+    if [ ${#i} -ge 15 ] ; then
+	printf '%s\n\t\t%s\n' "$i" "$opts"
+    else
+	printf '%s\t%s\n' "$i" "$opts"
+    fi
 done

From 62f2d160ac2faf31a551441b90b216dcf4c15d84 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 11:24:56 +1000
Subject: [PATCH 03/29] Tests - new NFS share checking tests

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit ea0db472c2ce138c723aa7e633b72ac9d8b8a6a3)
---
 .../eventscripts/simple/60.nfs.monitor.161.sh      | 13 +++++++++++++
 .../eventscripts/simple/60.nfs.monitor.162.sh      | 14 ++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh
 create mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh

diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh
new file mode 100755
index 00000000000..4abe68eaba7
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "2nd share missing"
+
+setup_nfs
+
+shares_missing "ERROR: nfs directory \"%s\" not available" 2
+
+required_result 1 "$MISSING_SHARES_TEXT"
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh
new file mode 100755
index 00000000000..865b1c635a6
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "2nd share missing, skipping share checks"
+
+setup_nfs
+export CTDB_NFS_SKIP_SHARE_CHECK="yes"
+
+shares_missing "ERROR: nfs directory \"%s\" not available" 2
+
+ok_null
+
+simple_test

From 7d7851dd45488f9851618b7281af7d427c265ec2 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 11:35:38 +1000
Subject: [PATCH 04/29] Tests - Change variable used to fake listening TCP
 ports.

Change from $FAKE_NETSTAT_TCP_LISTEN to $FAKE_TCP_LISTEN.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit f19ba38e382034fba769145d0bfc81b2c2941563)
---
 ctdb/tests/eventscripts/common.sh     | 6 +++---
 ctdb/tests/eventscripts/stubs/netstat | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index ab7abb9faca..f57df4eab78 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -109,7 +109,7 @@ tcp_port_down ()
 {
     for _i ; do
 	debug "Marking TCP port \"${_i}\" as not listening"
-	FAKE_NETSTAT_TCP_LISTEN=$(echo "$FAKE_NETSTAT_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
+	FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
     done
 }
 
@@ -303,7 +303,7 @@ setup_samba ()
 	export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
 	export CTDB_MANAGED_SERVICES="foo samba winbind bar"
 
-	export FAKE_NETSTAT_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
+	export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
 	export FAKE_WBINFO_FAIL="no"
     else
 	debug "Marking Samba services as down, not listening and not managed by CTDB"
@@ -321,7 +321,7 @@ setup_samba ()
 	unset CTDB_MANAGES_SAMBA
 	unset CTDB_MANAGES_WINBIND
 
-	export FAKE_NETSTAT_TCP_LISTEN=""
+	export FAKE_TCP_LISTEN=""
 	export FAKE_WBINFO_FAIL="yes"
     fi
 
diff --git a/ctdb/tests/eventscripts/stubs/netstat b/ctdb/tests/eventscripts/stubs/netstat
index cf0656651aa..5a3e3c7e77e 100755
--- a/ctdb/tests/eventscripts/stubs/netstat
+++ b/ctdb/tests/eventscripts/stubs/netstat
@@ -8,7 +8,7 @@ usage ()
 Usage: $prog [ -t | --unix ] [ -n ] [ -a ] [ -l ]
 
 A fake netstat stub that prints items depending on the variables
-FAKE_NETSTAT_TCP_ESTABLISHED, FAKE_NETSTAT_TCP_LISTEN,
+FAKE_NETSTAT_TCP_ESTABLISHED, FAKE_TCP_LISTEN,
 FAKE_NETSTAT_UNIX_LISTEN, depending on command-line options.
 
 Note that -n is ignored.
@@ -73,7 +73,7 @@ if $tcp ; then
     done
 
     if $all || $listen ; then
-	for i in $FAKE_NETSTAT_TCP_LISTEN ; do
+	for i in $FAKE_TCP_LISTEN ; do
 	    printf "$tcp_fmt" $i "0.0.0.0:*" "LISTEN"
 	done
     fi

From d2527e3e5bad11c3946799c4e9609d269309d00b Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 13:53:05 +1000
Subject: [PATCH 05/29] Tests - add eventscripts testing stub for sleep
 command.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit aedff41ee1df0a1a573476844993fa8eda2f34bf)
---
 ctdb/tests/eventscripts/stubs/sleep | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/stubs/sleep

diff --git a/ctdb/tests/eventscripts/stubs/sleep b/ctdb/tests/eventscripts/stubs/sleep
new file mode 100755
index 00000000000..adb63cdf41d
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/sleep
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ "$REALLY_SLEEP" = "yes" ] ; then
+    /bin/sleep "$@"
+else
+    :
+fi

From 21c17cc1005269b07e33c3b976fde16ede838621 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 13:54:20 +1000
Subject: [PATCH 06/29] Tests - add hooks to simulate ctdb commands that aren't
 implemented

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit e9bdfd43af3ed28a7f69a36e3fb037f252a23869)
---
 ctdb/tests/eventscripts/stubs/ctdb | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index ce06a35cbd8..4806cc78992 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -2,6 +2,8 @@
 
 prog="ctdb"
 
+not_implemented_exit_code=1
+
 usage ()
 {
     cat >&2 <<EOF
@@ -13,12 +15,10 @@ FAKE_CTDB_PNN (default 0) depending on command-line options.
 Note that -Y is ignored.
 
 EOF
-    exit 1
+    exit $not_implemented_exit_code
 }
 
-# $POSIXLY_CORRECT means that the command passed to onnode can take
-# options and getopt won't reorder things to make them options to this
-# script.
+# Don't set $POSIXLY_CORRECT here.
 _temp=$(getopt -n "$prog" -o "Yvh" -l help -- "$@") || \
     usage
 
@@ -51,6 +51,13 @@ setup_pstore ()
     mkdir -p "$pstore_dir"
 }
 
+# For testing backward compatibility...
+for i in $CTDB_NOT_IMPLEMENTED ; do
+    if [ "$i" = "$1" ] ; then
+	usage
+    fi
+done
+
 case "$1" in
     ip)
 	# NOTE: all nodes share the same public addresses file.

From 6eb339e2c73e4944e59be122de9a616ed68598eb Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 13:54:49 +1000
Subject: [PATCH 07/29] Tests - add getdebug and checktcpport to ctdb
 eventscripts stub

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 9c6fd7d21c99e605e20620c5c6b4e695c02af771)
---
 ctdb/tests/eventscripts/stubs/ctdb | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 4806cc78992..24dfff855f2 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -166,8 +166,35 @@ case "$1" in
 		exit 1
 	esac
 	;;
+    getdebug)
+	case "$CTDB_DEBUGLEVEL" in
+	    -3) _t="EMERG"  ;;
+	    -2) _t="ALERT"  ;;
+	    -1) _t="CRIT"   ;;
+	    0) _t="ERR"     ;;
+	    1) _t="WARNING" ;;
+	    2) _t="NOTICE"  ;;
+	    3) _t="INFO"    ;;
+	    4) _t="DEBUG"   ;;
+	    *) _t="ERR"     ;;
+	esac
+
+	cat<<EOF
+:Name:Level:
+:${_t}:${CTDB_DEBUGLEVEL}:
+EOF
+	;;
+    checktcpport)
+	for _i in $FAKE_TCP_LISTEN ; do
+	    if [ "$2" = "${_i##*:}" ] ; then
+		exit 98
+	    fi
+	done
+
+	exit 0
+	;;
     *)
 	echo "${prog}: command \"$1\" not implemented in stub"
-	exit 1
+	usage
 esac
 

From 7b2e79ab7f5c44ba70e8ab0632bdba17226a530f Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 19 Aug 2011 16:51:08 +1000
Subject: [PATCH 08/29] Tests - eventscripts - stop timeouts waiting for
 backgrounded testparm

Not sleeping at all speeds up the tests.  However, it can also cause
timeouts.  Therefore, every time sleep is run we force the stub to do
a short 0.1s sleep instead of whatever is specified.  This should be
enough to avoid races.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 59a1d54bcc15058f7b69e8596b32b26427a3ae57)
---
 ctdb/tests/eventscripts/common.sh   | 5 +++++
 ctdb/tests/eventscripts/stubs/sleep | 4 +++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index f57df4eab78..a2bcd830c92 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -305,6 +305,11 @@ setup_samba ()
 
 	export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
 	export FAKE_WBINFO_FAIL="no"
+
+	# Some things in 50.samba are backgrounded and waited for.  If
+	# we don't sleep at all then timeouts can happen.  This avoids
+	# that...  :-)
+	export FAKE_SLEEP_FORCE=0.1
     else
 	debug "Marking Samba services as down, not listening and not managed by CTDB"
         # Get into known state.
diff --git a/ctdb/tests/eventscripts/stubs/sleep b/ctdb/tests/eventscripts/stubs/sleep
index adb63cdf41d..e4542444de7 100755
--- a/ctdb/tests/eventscripts/stubs/sleep
+++ b/ctdb/tests/eventscripts/stubs/sleep
@@ -1,7 +1,9 @@
 #!/bin/sh
 
-if [ "$REALLY_SLEEP" = "yes" ] ; then
+if [ "$FAKE_SLEEP_REALLY" = "yes" ] ; then
     /bin/sleep "$@"
+elif [ -n "$FAKE_SLEEP_FORCE" ] ; then
+    /bin/sleep "$FAKE_SLEEP_FORCE"
 else
     :
 fi

From e88df4289f84f06c5e35d5cecc2ab003f9e1e095 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 15:56:57 +1000
Subject: [PATCH 09/29] Tests - eventscripts - add an nmap stub

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 6ff9545a0fd58f0728b769c7b36d3b6ab7366321)
---
 ctdb/tests/eventscripts/stubs/nmap | 68 ++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/stubs/nmap

diff --git a/ctdb/tests/eventscripts/stubs/nmap b/ctdb/tests/eventscripts/stubs/nmap
new file mode 100755
index 00000000000..15eebe6901f
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/nmap
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+prog="nmap"
+
+usage ()
+{
+    cat >&2 <<EOF
+Usage: $prog -n -oG - -PS 127.0.0.1 -p <port>[,<port> ...]
+
+A fake nmap stub that prints items depending on the variable
+FAKE_TCP_LISTEN and the ports specified.
+
+Note that all options apart from -p are ignored.
+
+EOF
+    exit 1
+}
+
+ports=""
+
+parse_options ()
+{
+    _temp=$(getopt -n "$prog" -a -o "np:" -l help -l PS: -l oG: -- "$@")
+
+    [ $? != 0 ] && usage
+
+    eval set -- "$_temp"
+
+    while true ; do
+	case "$1" in
+	    -n) shift ;;
+	    --oG|--PS) shift 2 ;;
+	    -p) ports="${ports}${ports:+ }${2//,/ }" ; shift 2 ;;
+	    --) shift ; break ;;
+	    -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable.
+	esac
+    done
+
+    [ $# -gt 0 ] && usage
+
+    [ -n "$ports" ] || usage
+}
+
+# For printing out...
+args="$*"
+
+parse_options "$@"
+
+port_states=""
+
+for p in $ports ; do
+    pn=$(getent services "$p" | sed -e 's@[[:space:]].*@@')
+    for i in $FAKE_TCP_LISTEN ; do
+	lp="${i##*:}"
+	if [ "$p" = "$lp" ] ; then
+	    port_states="${port_states}${port_states:+, }${p}/open/tcp//${pn}///"
+	    continue 2
+	fi
+    done
+    port_states="${port_states}${port_states:+, }${p}/closed/tcp//${pn}///"
+done
+
+cat <<EOF
+# Nmap 5.21 scan initiated $(date) as: nmap $args
+Host: 127.0.0.1 ()	Status: Up
+Host: 127.0.0.1 ()	Ports: $port_states
+# Nmap done at $(date) -- 1 IP address (1 host up) scanned in 0.04 seconds
+EOF

From c9305f5ddaf50ba0c715247818c5f21a7928b203 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 15:58:23 +1000
Subject: [PATCH 10/29] Tests - eventscripts - add output for "not implemented"
 in ctdb stub

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 1d02e3c6a7fb5ad5edb21298deba415ba9519f3d)
---
 ctdb/tests/eventscripts/stubs/ctdb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 24dfff855f2..1c26a305882 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -54,6 +54,7 @@ setup_pstore ()
 # For testing backward compatibility...
 for i in $CTDB_NOT_IMPLEMENTED ; do
     if [ "$i" = "$1" ] ; then
+	echo "${prog}: command \"$i\" not implemented" >&2
 	usage
     fi
 done

From 58687f52be4636b5ad2432e349c82641fef5b4b3 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 15:59:20 +1000
Subject: [PATCH 11/29] Tests - eventscripts - ctdb default default level is 0.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit fcf40500c2158dfbe2390fad7bf0614cc575df14)
---
 ctdb/tests/eventscripts/stubs/ctdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 1c26a305882..8d6409363e0 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -168,7 +168,7 @@ case "$1" in
 	esac
 	;;
     getdebug)
-	case "$CTDB_DEBUGLEVEL" in
+	case "${CTDB_DEBUGLEVEL:-0}" in
 	    -3) _t="EMERG"  ;;
 	    -2) _t="ALERT"  ;;
 	    -1) _t="CRIT"   ;;

From 5574b3b6fed7c5e611b57faba796a64b41e6ccb6 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:07:36 +1000
Subject: [PATCH 12/29] Tests - eventscripts - add some output filtering

This allows $OUT_FILTER to be set to one or more sed commands to
filter eventscript output.  This allows expected output to be
generalised.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit a1d424045c53f5b148952eade6161af8248ed4df)
---
 ctdb/tests/eventscripts/common.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index a2bcd830c92..0c605e7f458 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -708,7 +708,13 @@ simple_test ()
     _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
 
-    if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+    if [ -n "$OUT_FILTER" ] ; then
+	_fout=$(echo "$_out" | eval sed -r $OUT_FILTER)
+    else
+	_fout="$_out"
+    fi
+
+    if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
 	_passed=true
     else
 	_passed=false
@@ -770,7 +776,13 @@ iterate_test ()
 	_out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1)
 	_rc=$?
 
-	if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+    if [ -n "$OUT_FILTER" ] ; then
+	_fout=$(echo "$_out" | eval sed -r $OUT_FILTER)
+    else
+	_fout="$_out"
+    fi
+
+	if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
 	    _passed=true
 	else
 	    _passed=false

From bda5f71c139bacfdd7ef315d52b24704080384ba Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:00:52 +1000
Subject: [PATCH 13/29] Tests - eventscripts - new function
 setup_nmap_output_filter()

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit ef6131ecb068f7d8402cc782359509ba54b5adb0)
---
 ctdb/tests/eventscripts/common.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 0c605e7f458..1a83811caee 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -177,6 +177,11 @@ ethtool_interfaces_up ()
     done
 }
 
+setup_nmap_output_filter ()
+{
+    OUT_FILTER="-e 's@^(DEBUG: # Nmap 5.21 scan initiated) .+ (as:)@\1 DATE \2@' -e 's@^(DEBUG: # Nmap done at) .+ (--)@\1 DATE \2@'"
+}
+
 ######################################################################
 
 # CTDB fakery

From 764b425a965eee0be6a2a9b91774590533ff7070 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:06:16 +1000
Subject: [PATCH 14/29] Tests - eventscripts - add a new ctdb_not_implemented()
 function

This allowed a single ctdb command to be defined as not-implemented
and provided the associated output from the ctdb stub in
$ctdb_not_implemented.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 3852c091f5e0ecba06b7131ca04b73a0cd632f23)
---
 ctdb/tests/eventscripts/common.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 1a83811caee..235cdaf8868 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -285,6 +285,19 @@ ctdb_get_1_public_address ()
     echo "${_addrs%% *}" | sed -r -e 's#(.*)/(.*)@(.*)#\3 \1 \2#g'
 }
 
+ctdb_not_implemented ()
+{
+    export CTDB_NOT_IMPLEMENTED="$1"
+    ctdb_not_implemented="\
+DEBUG: ctdb: command \"$1\" not implemented
+DEBUG: Usage: ctdb [-Y] cmd
+DEBUG: 
+DEBUG: A fake CTDB stub that prints items depending on the variables
+DEBUG: FAKE_CTDB_PNN (default 0) depending on command-line options.
+DEBUG: 
+DEBUG: Note that -Y is ignored."
+}
+
 ######################################################################
 
 # Samba fakery

From 7fc7ebb5acf36aff258f914b522e1345802a5413 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:10:44 +1000
Subject: [PATCH 15/29] Tests - eventscripts - new Samba tests to test TCP port
 checking

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 9e3e6ab3edf0f79d5973eb39ed843d3c369c665d)
---
 .../simple/50.samba.monitor.107.sh            | 18 +++++++++++
 .../simple/50.samba.monitor.108.sh            | 15 ++++++++++
 .../simple/50.samba.monitor.109.sh            | 30 +++++++++++++++++++
 3 files changed, 63 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh
 create mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh
 create mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh

diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh
new file mode 100755
index 00000000000..e4151f37432
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, default tcp checker, debug"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+ERROR: samba tcp port 139 is not responding
+DEBUG: "ctdb checktcpport 139" was able to bind to port
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh
new file mode 100755
index 00000000000..d06a16f90a4
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport not implemented"
+
+# TODO: create nmap stub
+export CTDB_NOT_IMPLEMENTED="checktcpport"
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 "ERROR: samba tcp port 139 is not responding"
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
new file mode 100755
index 00000000000..b9137413251
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport not implemented, debug"
+
+ctdb_not_implemented "checktcpport"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+DEBUG: ctdb_check_ports - checker ctdb not implemented
+DEBUG: output from checker was:
+DEBUG: ctdb checktcpport 445 (exited with 0) with output:
+$ctdb_not_implemented
+ERROR: samba tcp port 139 is not responding
+DEBUG: nmap -n -oG - -PS 127.0.0.1 -p 445,139 shows this output:
+DEBUG: # Nmap 5.21 scan initiated DATE as: nmap -n -oG - -PS 127.0.0.1 -p 445,139
+DEBUG: Host: 127.0.0.1 ()	Status: Up
+DEBUG: Host: 127.0.0.1 ()	Ports: 445/open/tcp//microsoft-ds///, 139/closed/tcp//netbios-ssn///
+DEBUG: # Nmap done at DATE -- 1 IP address (1 host up) scanned in 0.04 seconds
+EOF
+
+simple_test

From fe730830a7c994d04824bfd7e0f909b008573880 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:14:55 +1000
Subject: [PATCH 16/29] Tests - eventscripts - nmap and netstat stubs can
 pretend they weren't found

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 47f2843b4db767bcc724d8f972e0c2a850d5826f)
---
 ctdb/tests/eventscripts/stubs/netstat | 7 +++++++
 ctdb/tests/eventscripts/stubs/nmap    | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/ctdb/tests/eventscripts/stubs/netstat b/ctdb/tests/eventscripts/stubs/netstat
index 5a3e3c7e77e..f46c9365193 100755
--- a/ctdb/tests/eventscripts/stubs/netstat
+++ b/ctdb/tests/eventscripts/stubs/netstat
@@ -2,6 +2,13 @@
 
 prog="netstat"
 
+# Pretty that we're the shell and that this command could not be
+# found.
+if [ "$FAKE_NETSTAT_NOT_FOUND" = "yes" ] ; then
+    echo "sh: ${prog}: command not found" >&2
+    exit 127
+fi
+
 usage ()
 {
     cat >&2 <<EOF
diff --git a/ctdb/tests/eventscripts/stubs/nmap b/ctdb/tests/eventscripts/stubs/nmap
index 15eebe6901f..f01fe32d9c4 100755
--- a/ctdb/tests/eventscripts/stubs/nmap
+++ b/ctdb/tests/eventscripts/stubs/nmap
@@ -2,6 +2,13 @@
 
 prog="nmap"
 
+# Pretty that we're the shell and that this command could not be
+# found.
+if [ "$FAKE_NMAP_NOT_FOUND" = "yes" ] ; then
+    echo "sh: ${prog}: command not found" >&2
+    exit 127
+fi
+
 usage ()
 {
     cat >&2 <<EOF

From 9b8e25d03b911bb628f906025060397d73f131b3 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:37:22 +1000
Subject: [PATCH 17/29] Tests - eventscripts - new Samba TCP port checking test
 - no nmap

This one should fall back to netstat.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 0a08068f736efe56b4799e1f1f90539d257deb71)
---
 .../simple/50.samba.monitor.110.sh            | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh

diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
new file mode 100755
index 00000000000..ca30d16fba9
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport/nmap not implemented, debug"
+
+ctdb_not_implemented "checktcpport"
+export FAKE_NMAP_NOT_FOUND="yes"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+DEBUG: ctdb_check_ports - checker ctdb not implemented
+DEBUG: output from checker was:
+DEBUG: ctdb checktcpport 445 (exited with 0) with output:
+$ctdb_not_implemented
+DEBUG: ctdb_check_ports - checker nmap not implemented
+DEBUG: output from checker was:
+DEBUG: sh: nmap: command not found
+ERROR: samba tcp port 139 is not responding
+DEBUG: netstat -l -t -n shows this output:
+DEBUG: Active Internet connections (servers only)
+DEBUG: Proto Recv-Q Send-Q Local Address           Foreign Address         State
+DEBUG: tcp        0      0 0.0.0.0:445             0.0.0.0:*               LISTEN
+EOF
+
+simple_test

From 5de459f94e2f4420ec60f16f669ff8443bb2c24f Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 22 Aug 2011 16:41:01 +1000
Subject: [PATCH 18/29] Tests - eventscripts - TCP port checking, no working
 checkers

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit b231aa629b909a7dfd071236c7e730e3cc4a1a54)
---
 .../simple/50.samba.monitor.111.sh            | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh

diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh
new file mode 100755
index 00000000000..4651290663a
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport/nmap/netstat not implemented"
+
+ctdb_not_implemented "checktcpport"
+export FAKE_NMAP_NOT_FOUND="yes"
+export FAKE_NETSTAT_NOT_FOUND="yes"
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 127 <<EOF
+INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS="ctdb nmap netstat"
+EOF
+
+simple_test

From 857e144a97e7a86cedaee3fcb5f519b802f9cf16 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 23 Aug 2011 13:52:42 +1000
Subject: [PATCH 19/29] Tests - eventscripts - Samba TCP port checking fixes

Expect "ctdb checktcpport" to exit with 1 if not implemented.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 20885d592040eb3945524c558b593a1d50a57b43)
---
 ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh | 2 +-
 ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
index b9137413251..4bc854b623e 100755
--- a/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
@@ -17,7 +17,7 @@ tcp_port_down 139
 required_result 1 <<EOF
 DEBUG: ctdb_check_ports - checker ctdb not implemented
 DEBUG: output from checker was:
-DEBUG: ctdb checktcpport 445 (exited with 0) with output:
+DEBUG: ctdb checktcpport 445 (exited with 1) with output:
 $ctdb_not_implemented
 ERROR: samba tcp port 139 is not responding
 DEBUG: nmap -n -oG - -PS 127.0.0.1 -p 445,139 shows this output:
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
index ca30d16fba9..80264cc24e3 100755
--- a/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
@@ -18,7 +18,7 @@ tcp_port_down 139
 required_result 1 <<EOF
 DEBUG: ctdb_check_ports - checker ctdb not implemented
 DEBUG: output from checker was:
-DEBUG: ctdb checktcpport 445 (exited with 0) with output:
+DEBUG: ctdb checktcpport 445 (exited with 1) with output:
 $ctdb_not_implemented
 DEBUG: ctdb_check_ports - checker nmap not implemented
 DEBUG: output from checker was:

From dcd37aadefa9164c5b5179d6d5f69f6737ccbe9b Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 23 Aug 2011 13:53:39 +1000
Subject: [PATCH 20/29] Tests - evenscripts - add symlink to ctdb.sysconfig

Some of the tests expect the default to be

  CTDB_SERVICE_AUTOSTARTSTOP=yes

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit f7de037e3be2e9910bee644dd1e93bae4912a194)
---
 ctdb/tests/eventscripts/etc/sysconfig/ctdb | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 ctdb/tests/eventscripts/etc/sysconfig/ctdb

diff --git a/ctdb/tests/eventscripts/etc/sysconfig/ctdb b/ctdb/tests/eventscripts/etc/sysconfig/ctdb
new file mode 120000
index 00000000000..1c6e2b9f05a
--- /dev/null
+++ b/ctdb/tests/eventscripts/etc/sysconfig/ctdb
@@ -0,0 +1 @@
+../../../../config/ctdb.sysconfig
\ No newline at end of file

From e4ba51effdc7945e2185a9071881dd55734ca262 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 13:35:43 +1000
Subject: [PATCH 21/29] Tests - eventscripts - remove undefined argument in
 some simple_test calls

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit c490024aba39cf94cbc9f8f41ecb3d336b1c82a8)
---
 ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh | 2 +-
 ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh b/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
index fe2afd1d9fb..0f8b1a53c63 100755
--- a/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
+++ b/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
@@ -8,4 +8,4 @@ setup_vsftpd "down"
 
 ok_null
 
-simple_test $cmd
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh b/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
index c46a45c0c75..0a9ceac428e 100755
--- a/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
+++ b/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
@@ -8,4 +8,4 @@ setup_httpd "down"
 
 ok_null
 
-simple_test $cmd
+simple_test

From 1a8d5e0ec7677cbd7a9998ad38565f137cf67770 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 14:16:28 +1000
Subject: [PATCH 22/29] Tests - eventscripts - add die() function and use it

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 0446eaa7798fd0ddf8d8f067c56d19892a13a317)
---
 ctdb/tests/eventscripts/common.sh | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 235cdaf8868..070aa45a059 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -1,5 +1,8 @@
 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
 
+# Print a message and exit.
+die () { echo "$@" >&2 ; exit 1 ; }
+
 # Augment PATH with relevant stubs/ directories.  We do this by actually
 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
 # prepending that to $PATH in rc.local to avoid the PATH reset in
@@ -30,8 +33,7 @@ if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc" ] ; then
 elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc" ] ; then
     CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_DIR}/etc"
 else
-    echo "Unable to set \$CTDB_ETCDIR" >&2
-    exit 1
+    die "Unable to set \$CTDB_ETCDIR"
 fi
 export CTDB_ETCDIR
 
@@ -42,8 +44,7 @@ elif [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" ] ; then
 elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" ] ; then
     CTDB_BASE="${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb"
 else
-    echo "Unable to set \$CTDB_BASE" >&2
-    exit 1
+    die "Unable to set \$CTDB_BASE"
 fi
 export CTDB_BASE
 
@@ -426,9 +427,7 @@ rpc_services_up ()
 	    rquotad)  _t="1:2" ;;
 	    nlockmgr) _t="3:4" ;;
 	    status)   _t="1:1" ;;
-	    *)
-		echo "Internal error - unsupported RPC service \"${_i}\""
-		exit 1
+	    *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
 	esac
 
 	FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
@@ -556,8 +555,7 @@ Starting nfslock: OK"
 setup_vsftpd ()
 {
     if [ "$1" != "down" ] ; then
-	echo "setup_vsftpd up not implemented!!!"
-	exit 1
+	die "setup_vsftpd up not implemented!!!"
     else
 	debug "Setting up VSFTPD environment: service down, not managed by CTDB"
 
@@ -576,8 +574,7 @@ setup_vsftpd ()
 setup_httpd ()
 {
     if [ "$1" != "down" ] ; then
-	echo "setup_httpd up not implemented!!!"
-	exit 1
+	die "setup_httpd up not implemented!!!"
     else
 	debug "Setting up HTTPD environment: service down, not managed by CTDB"
 

From 3167f50b6c978dda9175b740eca1487fba161ac0 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 14:19:09 +1000
Subject: [PATCH 23/29] Tests - eventscripts - add extra filename format for
 multi-event tests

$event may not be set so we need to test for it.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 80f14e590242b14829c6309ee908af0f2d33be1e)
---
 ctdb/tests/eventscripts/common.sh | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 070aa45a059..4c80a95868d 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -598,16 +598,27 @@ define_test ()
     desc="$1"
 
     _f="$0"
-    _f="${_f#./}"      # strip leading ./
-    _f="${_f#simple/}" # strip leading simple/
-    _f="${_f%%/*}"     # if subdir, strip off file
-    _f="${_f%.sh}"     # strip off .sh suffix if any
+    _f="${_f#./}"          # strip leading ./
+    _f="${_f#simple/}"     # strip leading simple/
+    _f="${_f#multievent/}" # strip leading multievent/
+    _f="${_f%%/*}"         # if subdir, strip off file
+    _f="${_f%.sh}"         # strip off .sh suffix if any
 
-    # Remaining format should be NN.service.event.NNN:
+    # Remaining format should be NN.service.event.NNN or NN.service.NNN:
     _num="${_f##*.}"
     _f="${_f%.*}"
-    event="${_f##*.}"
-    script="${_f%.*}"
+    case "$_f" in
+	*.*.*)
+	    script="${_f%.*}"
+	    event="${_f##*.}"
+	    ;;
+	*.*)
+	    script="$_f"
+	    unset event
+	    ;;
+	*)
+	    die "Internal error - unknown testcase filename format"
+    esac
 
     printf "%-14s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
 }
@@ -719,6 +730,8 @@ EOF
 # useful for debugging.
 simple_test ()
 {
+    [ -n "$event" ] || die 'simple_test: $event not set'
+
     echo "Running \"${CTDB_BASE}/events.d/$script $event\""
     _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
@@ -759,6 +772,8 @@ simple_test ()
 # iteration.
 iterate_test ()
 {
+    [ -n "$event" ] || die 'simple_test: $event not set'
+
     args=""
     if [ "$1" = "--" ] ; then
 	shift

From af1a0245b6d39d2ff6a4be40fa912adb79a70b7d Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 14:20:38 +1000
Subject: [PATCH 24/29] Tests - eventscripts - output format tweaks

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 8f8dc5636436db28ce7d0edd706a4adf2624e71b)
---
 ctdb/tests/eventscripts/common.sh | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 4c80a95868d..2d91740f0e0 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -668,25 +668,24 @@ result_print ()
 	if [ -n "$_iteration" ] ; then
 	    cat <<EOF
 
-##################################################
-##################################################
+==================================================
 Iteration $_iteration
 EOF
 	fi
 
 cat <<EOF
-##################################################
+--------------------------------------------------
 Output (Exit status: ${_rc}):
-##################################################
+--------------------------------------------------
 $_out
 EOF
     fi
 
     if ! $_passed ; then
 	cat <<EOF
-##################################################
+--------------------------------------------------
 Required output (Exit status: ${required_rc}):
-##################################################
+--------------------------------------------------
 $required_output
 EOF
     fi
@@ -699,20 +698,19 @@ result_footer ()
     if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then
 
 	cat <<EOF
-##################################################
+--------------------------------------------------
 CTDB_BASE="$CTDB_BASE"
 CTDB_ETCDIR="$CTDB_ETCDIR"
 ctdb client is "$(which ctdb)"
-##################################################
+--------------------------------------------------
 EOF
     fi
 
-    echo
-
     if $_passed ; then
 	echo "PASSED"
 	return 0
     else
+	echo
 	echo "FAILED"
 	return 1
     fi
@@ -732,7 +730,7 @@ simple_test ()
 {
     [ -n "$event" ] || die 'simple_test: $event not set'
 
-    echo "Running \"${CTDB_BASE}/events.d/$script $event\""
+    echo "Running \"$script $event $*\""
     _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
 
@@ -788,7 +786,7 @@ iterate_test ()
     _setup_default="$2"
     shift 2
 
-    echo "Running $_repeats iterations of \"${CTDB_BASE}/events.d/$script $event\" $args"
+    echo "Running $_repeats iterations of \"$script $event\" $args"
 
     _result=true
 

From c8e87d9f003c45fb5d041f5b5394b2d46e69106d Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 14:21:02 +1000
Subject: [PATCH 25/29] Tests - eventscripts - new function simple_test_event()

Just like simple_test() but 1st arg is the event name.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 69e240e15488e228c0232fdc8804af778cb4e6fc)
---
 ctdb/tests/eventscripts/common.sh | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 2d91740f0e0..713e426455d 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -750,6 +750,17 @@ simple_test ()
     result_footer "$_passed"
 }
 
+simple_test_event ()
+{
+    # If something has previously failed then don't continue.
+    : ${_passed:=true}
+    $_passed || return 1
+
+    event="$1" ; shift
+    echo "##################################################"
+    simple_test "$@"
+}
+
 # Run an eventscript iteratively.
 # - 1st argument is the number of iterations.
 # - 2nd argument is something to eval to do setup for every iteration.

From 84c67c378e331f0515ae203839bde98598eafa08 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 16:25:13 +1000
Subject: [PATCH 26/29] Tests - eventscripts - formatting tweak in
 simple_test()

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 698af79545986cf788686feb06c4677a10eec770)
---
 ctdb/tests/eventscripts/common.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 713e426455d..9a2859f3975 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -730,7 +730,7 @@ simple_test ()
 {
     [ -n "$event" ] || die 'simple_test: $event not set'
 
-    echo "Running \"$script $event $*\""
+    echo "Running \"$script $event${1:+ }$*\""
     _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
 

From 91d5edee80fd103827b12c867081ad8573aaef22 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Aug 2011 16:27:04 +1000
Subject: [PATCH 27/29] Tests - eventscripts - ctdb stub - implement
 scriptstatus, tweaks

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit edc34d01a1ad83b1335c976593a006166b196569)
---
 ctdb/tests/eventscripts/stubs/ctdb | 37 ++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 8d6409363e0..5b18473ec1d 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -11,10 +11,13 @@ Usage: $prog [-Y] cmd
 
 A fake CTDB stub that prints items depending on the variables
 FAKE_CTDB_PNN (default 0) depending on command-line options.
-
-Note that -Y is ignored.
-
 EOF
+    exit 1
+}
+
+not_implemented ()
+{
+    echo "${prog}: command \"$1\" not implemented in stub" >&2
     exit $not_implemented_exit_code
 }
 
@@ -25,10 +28,11 @@ _temp=$(getopt -n "$prog" -o "Yvh" -l help -- "$@") || \
 eval set -- "$_temp"
 
 verbose=false
+machine_readable=false
 
 while true ; do
     case "$1" in
-	-Y) shift ;;
+	-Y) machine_readable=true ; shift ;;
 	-v) verbose=true ; shift ;;
 	--) shift ; break ;;
 	-h|--help|*) usage ;; # * shouldn't happen, so this is reasonable.
@@ -54,8 +58,7 @@ setup_pstore ()
 # For testing backward compatibility...
 for i in $CTDB_NOT_IMPLEMENTED ; do
     if [ "$i" = "$1" ] ; then
-	echo "${prog}: command \"$i\" not implemented" >&2
-	usage
+	not_implemented "$i"
     fi
 done
 
@@ -194,8 +197,24 @@ EOF
 
 	exit 0
 	;;
+    scriptstatus)
+	$machine_readable || not_implemented "$1, without -Y"
+	[ "$2" != "all" ] || not_implemented "scriptstatus all"
+	# For now just assume everything is good.
+	_d1=$(date '+%s.%N')
+	_d2=$(date '+%s.%N')
+	echo ":Type:Name:Code:Status:Start:End:Error Output...:"
+	for _i in "$CTDB_BASE/events.d/"*.* ; do
+	    _b="${_i##*/}" # basename
+	    _status="OK"
+	    _code="0"
+	    if [ ! -x "$_i" ] ; then
+		_status="DISABLED"
+		_code="-8"
+	    fi
+	    echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}::"
+	done
+	;;
     *)
-	echo "${prog}: command \"$1\" not implemented in stub"
-	usage
+	not_implemented "$1"
 esac
-

From 0048849bd2c2c561e88f3d2898342f984ffb6ced Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 31 Aug 2011 17:27:05 +1000
Subject: [PATCH 28/29] Tests - eventscripts - allow "ctdb scriptstatus" output
 to be primed

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit f527ffb55b514b5cb4df3471a0e17ab71307e8a1)
---
 ctdb/tests/eventscripts/common.sh  | 24 +++++++++++++++++-------
 ctdb/tests/eventscripts/stubs/ctdb | 23 +++++++++++++++--------
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index 9a2859f3975..9003b396c20 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -238,6 +238,10 @@ setup_ctdb ()
     export FAKE_CTDB_NODES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/nodes-down"
     mkdir -p "$FAKE_CTDB_NODES_DOWN"
     rm -f "$FAKE_CTDB_NODES_DOWN"/*
+
+    export FAKE_CTDB_SCRIPTSTATUS="$EVENTSCRIPTS_TESTS_VAR_DIR/scriptstatus"
+    mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
+    rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
 }
 
 
@@ -290,13 +294,19 @@ ctdb_not_implemented ()
 {
     export CTDB_NOT_IMPLEMENTED="$1"
     ctdb_not_implemented="\
-DEBUG: ctdb: command \"$1\" not implemented
-DEBUG: Usage: ctdb [-Y] cmd
-DEBUG: 
-DEBUG: A fake CTDB stub that prints items depending on the variables
-DEBUG: FAKE_CTDB_PNN (default 0) depending on command-line options.
-DEBUG: 
-DEBUG: Note that -Y is ignored."
+DEBUG: ctdb: command \"$1\" not implemented in stub"
+}
+
+ctdb_fake_scriptstatus ()
+{
+    _code="$1"
+    _status="$2"
+    _err_out="$3"
+
+    _d1=$(date '+%s.%N')
+    _d2=$(date '+%s.%N')
+
+    echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
 }
 
 ######################################################################
diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 5b18473ec1d..155f518ddc9 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -201,18 +201,25 @@ EOF
 	$machine_readable || not_implemented "$1, without -Y"
 	[ "$2" != "all" ] || not_implemented "scriptstatus all"
 	# For now just assume everything is good.
-	_d1=$(date '+%s.%N')
-	_d2=$(date '+%s.%N')
 	echo ":Type:Name:Code:Status:Start:End:Error Output...:"
 	for _i in "$CTDB_BASE/events.d/"*.* ; do
+	    _d1=$(date '+%s.%N')
 	    _b="${_i##*/}" # basename
-	    _status="OK"
-	    _code="0"
-	    if [ ! -x "$_i" ] ; then
-		_status="DISABLED"
-		_code="-8"
+
+	    _f="$FAKE_CTDB_SCRIPTSTATUS/$_b"
+	    if [ -r "$_f" ] ; then
+		read _code _status _err_out <"$_f"
+	    else
+		_code="0"
+		_status="OK"
+		if [ ! -x "$_i" ] ; then
+		    _status="DISABLED"
+		    _code="-8"
+		fi
+		_err_out=""
 	    fi
-	    echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}::"
+	    _d2=$(date '+%s.%N')
+	    echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}:${_err_out}:"
 	done
 	;;
     *)

From 882fb55becee9e15befa6ad7b54807eb3252a1ef Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 31 Aug 2011 17:29:14 +1000
Subject: [PATCH 29/29] Tests - eventscripts - add some multievent tests

Running "./run_tests -s simple/* multievent/*" results in a stupid
summary.  Will need to work on that...

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 999c68948da1492ac4d728c6bf79ad4921d9e840)
---
 .../multievent/10.interface.001.sh            | 14 ++++++++++
 .../eventscripts/multievent/60.nfs.001.sh     | 21 +++++++++++++++
 .../eventscripts/multievent/60.nfs.002.sh     | 25 +++++++++++++++++
 .../eventscripts/multievent/60.nfs.003.sh     | 27 +++++++++++++++++++
 .../eventscripts/multievent/60.nfs.004.sh     | 27 +++++++++++++++++++
 .../eventscripts/multievent/60.nfs.005.sh     | 27 +++++++++++++++++++
 .../eventscripts/multievent/60.nfs.006.sh     | 21 +++++++++++++++
 7 files changed, 162 insertions(+)
 create mode 100755 ctdb/tests/eventscripts/multievent/10.interface.001.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.001.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.002.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.003.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.004.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.005.sh
 create mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.006.sh

diff --git a/ctdb/tests/eventscripts/multievent/10.interface.001.sh b/ctdb/tests/eventscripts/multievent/10.interface.001.sh
new file mode 100755
index 00000000000..5ef254c536f
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/10.interface.001.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, removeip"
+
+setup_ctdb
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+simple_test_event "releaseip" $public_address
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.001.sh b/ctdb/tests/eventscripts/multievent/60.nfs.001.sh
new file mode 100755
index 00000000000..a6608db83e2
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.001.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, ipreallocated -> reconfigure"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+EOF
+
+simple_test_event "ipreallocated"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.002.sh b/ctdb/tests/eventscripts/multievent/60.nfs.002.sh
new file mode 100755
index 00000000000..93f645a6185
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.002.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+# This currently assumes that ctdb scriptstatus will always return a
+# good status (when replaying).  That should change and we will need
+# to split this into 2 tests.
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.003.sh b/ctdb/tests/eventscripts/multievent/60.nfs.003.sh
new file mode 100755
index 00000000000..929eaf2ae53
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.003.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay error"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err="foo: bar error occurred"
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus 1 "ERROR" "$err"
+
+required_result 1 <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+$err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.004.sh b/ctdb/tests/eventscripts/multievent/60.nfs.004.sh
new file mode 100755
index 00000000000..addbf44c9c7
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.004.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay timedout"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err="waiting, waiting..."
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus -62 "TIMEDOUT" "$err"
+
+required_result 1 <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+[Replay of TIMEDOUT scriptstatus - note incorrect return code.] $err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.005.sh b/ctdb/tests/eventscripts/multievent/60.nfs.005.sh
new file mode 100755
index 00000000000..9e652013ca8
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.005.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay disabled"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err=""
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus -8 "DISABLED" "$err"
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+[Replay of DISABLED scriptstatus - note incorrect return code.] $err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.006.sh b/ctdb/tests/eventscripts/multievent/60.nfs.006.sh
new file mode 100755
index 00000000000..35f8b836e12
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.006.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "reconfigure (synthetic), twice"
+# This checks that the lock is released...
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err=""
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+EOF
+
+simple_test_event "reconfigure"
+simple_test_event "reconfigure"