tests/eventscripts: Tests for memory checking in 00.ctdb

... plus updates to test infrastructure to support.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 4a388fc6bf54636b7e1f6da8e6aa451cddd574f7)

ctdb/tests/eventscripts/00.ctdb.monitor.001.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, no checks enabled"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok_null
+
+simple_test

ctdb/tests/eventscripts/00.ctdb.monitor.002.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, good situation, all enabled"
+
+setup_memcheck
+
+CTDB_MONITOR_FREE_MEMORY="500"
+CTDB_MONITOR_FREE_MEMORY_WARN="1000"
+CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+
+ok_null
+
+simple_test

ctdb/tests/eventscripts/00.ctdb.monitor.003.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only swap check"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+
+ok <<EOF
+We are swapping:
+$FAKE_PROC_MEMINFO
+$(ps foobar)
+EOF
+
+simple_test

ctdb/tests/eventscripts/00.ctdb.monitor.004.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only memory warning"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY=""
+CTDB_MONITOR_FREE_MEMORY_WARN="500"
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok <<EOF
+WARNING: free memory is low - 468MB free <=  ${CTDB_MONITOR_FREE_MEMORY_WARN}MB (CTDB threshold)
+EOF
+
+simple_test

ctdb/tests/eventscripts/00.ctdb.monitor.005.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, only memory critical"
+
+setup_memcheck "bad"
+
+CTDB_MONITOR_FREE_MEMORY="500"
+CTDB_MONITOR_FREE_MEMORY_WARN=""
+CTDB_CHECK_SWAP_IS_NOT_USED="no"
+
+ok <<EOF
+CRITICAL: OOM - 468MB free <= ${CTDB_MONITOR_FREE_MEMORY}MB (CTDB threshold)
+CRITICAL: Shutting down CTDB!!!
+$FAKE_PROC_MEMINFO
+$(ps foobar)
+CTDB says BYE!
+EOF
+
+simple_test

ctdb/tests/eventscripts/etc-ctdb/rc.local

@@ -39,6 +39,9 @@ get_proc ()
 	*/stack)
 	    echo "[<ffffffff87654321>] fake_stack_trace_for_pid_${1}+0x0/0xff"
 	    ;;
+	meminfo)
+	    echo "$FAKE_PROC_MEMINFO"
+	    ;;
 	*)
 	    echo "get_proc: \"$1\" not implemented"
 	    exit 1

ctdb/tests/eventscripts/scripts/local.sh

@@ -311,6 +311,51 @@ setup_ctdb ()
     export CTDB_PARTIALLY_ONLINE_INTERFACES
 }
 
+setup_memcheck ()
+{
+    setup_ctdb
+
+    _swap_total="5857276"
+
+    if [ "$1" = "bad" ] ; then
+	_swap_free="   4352"
+	_mem_cached=" 112"
+	_mem_free=" 468"
+    else
+	_swap_free="$_swap_total"
+	_mem_cached="1112"
+	_mem_free="1468"
+    fi
+
+    export FAKE_PROC_MEMINFO="\
+MemTotal:        3940712 kB
+MemFree:          225268 kB
+Buffers:          146120 kB
+Cached:          1139348 kB
+SwapCached:        56016 kB
+Active:          2422104 kB
+Inactive:        1019928 kB
+Active(anon):    1917580 kB
+Inactive(anon):   523080 kB
+Active(file):     504524 kB
+Inactive(file):   496848 kB
+Unevictable:        4844 kB
+Mlocked:            4844 kB
+SwapTotal:       ${_swap_total} kB
+SwapFree:        ${_swap_free} kB
+..."
+
+    export FAKE_FREE_M="\
+             total       used       free     shared    buffers     cached
+Mem:          3848       3634        213          0        142       ${_mem_cached}
+-/+ buffers/cache:       2379       ${_mem_free}
+Swap:         5719        246       5473"
+
+    export CTDB_MONITOR_FREE_MEMORY
+    export CTDB_MONITOR_FREE_MEMORY_WARN
+    export CTDB_CHECK_SWAP_IS_NOT_USED
+}
+
 ctdb_get_interfaces ()
 {
     # The echo/subshell forces all the output onto 1 line.

ctdb/tests/eventscripts/stubs/ctdb

@@ -124,8 +124,8 @@ ip_reallocate ()
 		# Have non-zero flags
 		_this=0
 		for _j in "$FAKE_CTDB_STATE/node-state/"*"/$_i" ; do
-		    _t="${_j%/*}" # dirname
-		    _f="${_t%/*}" # basename
+		    _tf="${_j%/*}" # dirname
+		    _f="${_tf##*/}" # basename
 		    _this=$(( $_this | $_f ))
 		done
 	    else
@@ -224,6 +224,13 @@ ctdb_disable ()
 
 ######################################################################
 
+ctdb_shutdown ()
+{
+    echo "CTDB says BYE!"
+}
+
+######################################################################
+
 case "$1" in
     gettickles)
 	setup_tickles
@@ -322,5 +329,6 @@ case "$1" in
     enable)      ctdb_enable "$@";;
     disable)     ctdb_disable "$@";;
     moveip)      ctdb_moveip "$@";;
+    shutdown)    ctdb_shutdown "$@";;
     *) not_implemented "$1" ;;
 esac

ctdb/tests/eventscripts/stubs/free

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$1" = "-m" ] ; then
+    echo "$FAKE_FREE_M"
+    exit 0
+else
+    echo "free: not implemented - $*"
+    exit 1
+fi

ctdb/tests/eventscripts/stubs/ps

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+cat <<EOF
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
+root         3  0.0  0.0      0     0 ?        S    Aug28   0:43  \_ [ksoftirqd/0]
+...
+root         1  0.0  0.0   2976   624 ?        Ss   Aug28   0:07 init [2]  
+root       495  0.0  0.0   3888  1640 ?        Ss   Aug28   0:00 udevd --daemon
+...
+[MORE FAKE ps OUTPUT]
+EOF