diff --git a/ctdb/tests/complex/31_nfs_tickle.sh b/ctdb/tests/complex/31_nfs_tickle.sh new file mode 100755 index 00000000000..4961b7dae50 --- /dev/null +++ b/ctdb/tests/complex/31_nfs_tickle.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +test_info() +{ + cat </dev/null 2>&1" + +wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc" +src_socket="$out" +echo "Source socket is $src_socket" + +echo "Sleeping for MonitorInterval..." +sleep_for $monitor_interval + +try_command_on_node $test_node hostname +test_hostname=$out + +try_command_on_node -v 0 cat /gpfs/.ctdb/nfs-tickles/$test_hostname/$test_ip + +if [ "${out/${src_socket}/}" != "$out" ] ; then + echo "GOOD: NFS connection tracked OK in tickles file." +else + echo "BAD: Socket not tracked in NFS tickles file:" + testfailures=1 +fi + +filter="src host $test_ip and tcp src port $test_port and dst host ${src_socket%:*} and tcp dst port ${src_socket##*:} and tcp[tcpflags] & tcp-rst != 0" +tcpdump_start "$filter" + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled + +tcpdump_wait + +echo "GOOD: here's the tickle reset:" +tcpdump -n -r $tcpdump_filename 2>/dev/null + +echo "Expect a restart..." + +ctdb_test_exit diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh new file mode 100755 index 00000000000..40230c1530b --- /dev/null +++ b/ctdb/tests/complex/32_cifs_tickle.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +test_info() +{ + cat </dev/null 2>&1" + +wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc" +src_socket="$out" +echo "Source socket is $src_socket" + +# Right here we assume that Samba is able to register the tickle with +# CTDB faster than it takes us to wait for netstat to register the +# connection and then use onnode below to ask CTDB about it. + +try_command_on_node -v 0 ctdb gettickles $test_ip + +if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then + echo "GOOD: CIFS connection tracked OK by CTDB." +else + echo "BAD: Socket not tracked by CTDB." + testfailures=1 +fi + +filter="src host $test_ip and tcp src port $test_port and dst host ${src_socket%:*} and tcp dst port ${src_socket##*:} and tcp[tcpflags] & tcp-rst != 0" +tcpdump_start "$filter" + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled + +tcpdump_wait + +echo "GOOD: here's the tickle reset:" +tcpdump -n -r $tcpdump_filename 2>/dev/null + +echo "Expect a restart..." + +ctdb_test_exit diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index f1e141745c0..1930ae163a0 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -61,7 +61,7 @@ ctdb_test_exit () [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status - eval "$ctdb_test_exit_hook" + eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then @@ -79,6 +79,11 @@ ctdb_test_exit () test_exit } +ctdb_test_exit_hook_add () +{ + ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" +} + ctdb_test_run () { local name="$1" ; shift @@ -150,6 +155,14 @@ ctdb_test_init () trap "ctdb_test_exit" 0 } +ctdb_test_check_real_cluster () +{ + [ -n "$CTDB_TEST_REAL_CLUSTER" ] && return 0 + + echo "ERROR: This test must be run on a real/virtual cluster, not local daemons." + return 1 +} + ######################################## # Sets: $out @@ -401,6 +414,63 @@ wait_until_ips_are_on_nodeglob () wait_until 60 ips_are_on_nodeglob "$@" } +get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + out=$(netstat -tanp | + egrep "$pat" | + awk '{ print $4 }') + + [ -n "$out" ] +} + +wait_until_get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + echo "Waiting for ${prog} to establish connection to ${dst_socket}..." + + wait_until 5 get_src_socket "$@" +} + +# filename will be in $tcpdump_filename, pid in $tcpdump_pid +# By default, wait for 1 matching packet on any interface. +tcpdump_start () +{ + local filter="$1" + local count="${2:-1}" + local iface="${3:-any}" + + echo "Running tcpdump to capture ${count} packet(s) on interface ${iface}." + tcpdump_filename=$(mktemp) + ctdb_test_exit_hook_add "rm -f $tcpdump_filename" + tcpdump -s 1500 -w $tcpdump_filename -c "$count" -i "$iface" "$filter" & + tcpdump_pid=$! + ctdb_test_exit_hook_add "kill $tcpdump_pid >/dev/null 2>&1" + echo "Waiting for tcpdump output file to be initialised..." + wait_until 10 test -f $tcpdump_filename + sleep_for 1 +} + +not () +{ + ! "$@" +} + +tcpdump_wait () +{ + echo "Waiting for tcpdump to complete..." + wait_until 5 not kill -0 $tcpdump_pid >/dev/null 2>&1 +} + ####################################### daemons_stop ()