ctdb-tests: Unit tests for statd-callout

With improvements to unit test infrastructure to support.  This
includes linking the real statd-callout into etc-ctdb/ in place of the
placeholder script.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

ctdb/tests/eventscripts/etc-ctdb/statd-callout

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-# For now, always succeed.
-
-exit 0

ctdb/tests/eventscripts/etc-ctdb/statd-callout

@@ -0,0 +1 @@
+../../../config/statd-callout

ctdb/tests/eventscripts/scripts/local.sh

@@ -538,6 +538,65 @@ EOF
 
 ######################################################################
 
+ctdb_catdb_format_pairs ()
+{
+    _count=0
+
+    while read _k _v ; do
+	_kn=$(echo -n "$_k" | wc -c)
+	_vn=$(echo -n "$_v" | wc -c)
+	cat <<EOF
+key(${_kn}) = "${_k}"
+dmaster: 0
+rsn: 1
+data(${_vn}) = "${_v}"
+
+EOF
+	_count=$(($_count + 1))
+    done
+
+    echo "Dumped ${_count} records"
+}
+
+check_ctdb_tdb_statd_state ()
+{
+    ctdb_get_my_public_addresses |
+    while read _x _sip _x ; do
+	for _cip ; do
+	    echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
+	done
+    done |
+    ctdb_catdb_format_pairs | {
+	ok
+	simple_test_command ctdb catdb ctdb.tdb
+    }
+}
+
+check_statd_callout_smnotify ()
+{
+    _state_even=$(( $(date '+%s') / 2 * 2))
+    _state_odd=$(($_state_even + 1))
+
+    nfs_load_config
+
+    ctdb_get_my_public_addresses |
+    while read _x _sip _x ; do
+	for _cip ; do
+	    cat <<EOF
+--client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
+--client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
+--client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
+--client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
+EOF
+	done
+    done | {
+	ok
+	simple_test_event "notify"
+    }
+}
+
+######################################################################
+
 setup_ctdb_natgw ()
 {
     debug "Setting up NAT gateway"
@@ -810,6 +869,19 @@ rpc_services_up ()
     done
 }
 
+
+nfs_load_config ()
+{
+    _etc="$CTDB_ETCDIR" # shortcut for readability
+    for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
+	if [ -r "$_c" ] ; then
+	    . "$_c"
+	    break
+	fi
+    done
+}
+
+
 # Set the required result for a particular RPC program having failed
 # for a certain number of iterations.  This is probably still a work
 # in progress.  Note that we could hook aggressively
@@ -826,13 +898,7 @@ rpc_set_service_failure_response ()
     # the flexibility to set the number of failures.
     _numfails="${2:-${iteration}}"
 
-    _etc="$CTDB_ETCDIR" # shortcut for readability
-    for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
-	if [ -r "$_c" ] ; then
-	    . "$_c"
-	    break
-	fi
-    done
+    nfs_load_config
 
     # A handy newline.  :-)
     _nl="

ctdb/tests/eventscripts/statd-callout.001.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "single add-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state "192.168.123.45"

ctdb/tests/eventscripts/statd-callout.002.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+simple_test_event "add-client" "192.168.123.46"
+
+check_ctdb_tdb_statd_state "192.168.123.45" "192.168.123.46"

ctdb/tests/eventscripts/statd-callout.003.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "add-client, del-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+simple_test_event "del-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state

ctdb/tests/eventscripts/statd-callout.004.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "single add-client, notify"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state "192.168.123.45"
+
+check_statd_callout_smnotify "192.168.123.45"
+
+check_ctdb_tdb_statd_state

ctdb/tests/eventscripts/statd-callout.005.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client to different nodes, notify on 1"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+ok_null
+simple_test_event "add-client" "192.168.123.46"
+
+FAKE_CTDB_PNN=0
+
+check_statd_callout_smnotify "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+check_ctdb_tdb_statd_state "192.168.123.46"

ctdb/tests/eventscripts/statd-callout.006.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client to different nodes, notify on both"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+ok_null
+simple_test_event "add-client" "192.168.123.46"
+
+FAKE_CTDB_PNN=0
+
+check_statd_callout_smnotify "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+check_statd_callout_smnotify "192.168.123.46"
+
+check_ctdb_tdb_statd_state

ctdb/tests/eventscripts/stubs/id

@@ -0,0 +1,3 @@
+#!/bin/sh
+# Make statd-callout happy
+echo 0

ctdb/tests/eventscripts/stubs/smnotify

@@ -0,0 +1,3 @@
+#!/bin/sh
+# echo args
+echo "$*"

ctdb/wscript

@@ -628,7 +628,8 @@ def build(bld):
     test_eventscript_links = [
         'events.d',
         'functions',
-        'nfs-rpc-checks.d'
+        'nfs-rpc-checks.d',
+        'statd-callout'
     ]
 
     test_link_dir = os.path.join(bld.env.CTDB_TEST_DATADIR,