ctdb-scripts: Add default system memory usage warnings

CTDB should warn by default if too much system memory or swap is used.

The tests have also been tweaked.  In particular, the filesystem-only
tests need to initialise the memory information to avoid errors where
meminfo isn't set.

Document the defaults, warning against disabling them.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

ctdb/config/events.d/05.system

@@ -103,9 +103,12 @@ dump_memory_info ()
 
 monitor_memory_usage ()
 {
-    if [ -z "$CTDB_MONITOR_MEMORY_USAGE" -a \
-	 -z "$CTDB_MONITOR_SWAP_USAGE" ] ; then
-	return
+    # Defaults
+    if [ -z "$CTDB_MONITOR_MEMORY_USAGE" ] ; then
+	CTDB_MONITOR_MEMORY_USAGE=80
+    fi
+    if [ -z "$CTDB_MONITOR_SWAP_USAGE" ] ; then
+	CTDB_MONITOR_SWAP_USAGE=25
     fi
 
     _meminfo=$(get_proc "meminfo")

ctdb/doc/ctdbd.conf.5.xml

@@ -1280,8 +1280,16 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
       <para>
 	CTDB can experience seemingly random (performance and other)
 	issues if system resources become too constrained.  Options in
-	this section can be enabled to allow certain system resources to
-	be checked.
+	this section can be enabled to allow certain system resources
+	to be checked.  They allows warnings to be logged and nodes to
+	be marked unhealthy when system resource usage reaches the
+	configured thresholds.
+      </para>
+
+      <para>
+	Some checks are enabled by default.  It is recommended that
+	these checks remain enabled or are augmented by extra checks.
+	There is no supported way of completely disabling the checks.
       </para>
 
       <refsect3>
@@ -1331,7 +1339,8 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
 	      left blank, meaning that check will be omitted.
 	    </para>
 	    <para>
-	      No default.
+	      Default is 80, so warnings will be logged when memory
+	      usage reaches 80%.
 	    </para>
 	  </listitem>
 	</varlistentry>
@@ -1349,7 +1358,8 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
 	      left blank, meaning that check will be omitted.
 	    </para>
 	    <para>
-	      No default.
+	      Default is 25, so warnings will be logged when swap
+	      usage reaches 25%.
 	    </para>
 	  </listitem>
 	</varlistentry>

ctdb/tests/eventscripts/05.system.monitor.001.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, error situation, no checks enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE=""
 setup_fscheck 100
 ok_null

ctdb/tests/eventscripts/05.system.monitor.002.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, good situation, 1 error check enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var::80"
 setup_fscheck
 ok_null

ctdb/tests/eventscripts/05.system.monitor.003.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, error situation, 1 error check enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var::80"
 setup_fscheck 90
 required_result 1 <<EOF

ctdb/tests/eventscripts/05.system.monitor.004.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, warn situation, only error check enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var::80"
 setup_fscheck 70
 ok_null

ctdb/tests/eventscripts/05.system.monitor.005.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, warn situation, both checks enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var:80:90"
 setup_fscheck 85
 ok <<EOF

ctdb/tests/eventscripts/05.system.monitor.006.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, error situation, both checks enabled"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var:80:90"
 setup_fscheck 95
 required_result 1 <<EOF

ctdb/tests/eventscripts/05.system.monitor.007.sh

@@ -4,6 +4,8 @@
 
 define_test "Filesystem use check, good situation, both checks enabled, multiple filesystems"
 
+setup_memcheck
+
 CTDB_MONITOR_FILESYSTEM_USAGE="/var:80:90 /:90:95"
 setup_fscheck
 ok_null

ctdb/tests/eventscripts/05.system.monitor.011.sh

@@ -2,13 +2,15 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, bad situation, no checks enabled"
+define_test "Memory check, bad situation, default checks enabled"
 
 setup_memcheck 100 100
 
 CTDB_MONITOR_MEMORY_USAGE=""
 CTDB_MONITOR_SWAP_USAGE=""
 
-ok_null
-
+ok <<EOF
+WARNING: System memory utilization 100% >= threshold 80%
+WARNING: System swap utilization 100% >= threshold 25%
+EOF
 simple_test

ctdb/tests/eventscripts/05.system.monitor.012.sh

@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, good situation, all enabled"
+define_test "Memory check, good situation, all memory checks enabled"
 
 setup_memcheck
 

ctdb/tests/eventscripts/05.system.monitor.013.sh

@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, bad situation, only swap check"
+define_test "Memory check, bad situation, custom swap critical"
 
 setup_memcheck 100 90
 
@@ -10,6 +10,7 @@ CTDB_MONITOR_MEMORY_USAGE=""
 CTDB_MONITOR_SWAP_USAGE=":50"
 
 required_result 1 <<EOF
+WARNING: System memory utilization 100% >= threshold 80%
 ERROR: System swap utilization 90% >= threshold 50%
 $FAKE_PROC_MEMINFO
 $(ps foobar)

ctdb/tests/eventscripts/05.system.monitor.014.sh

@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, bad situation, only memory warning"
+define_test "Memory check, bad memory situation, custom memory warning"
 
 setup_memcheck 90 10
 

ctdb/tests/eventscripts/05.system.monitor.015.sh

@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, bad situation, only memory critical"
+define_test "Memory check, bad situation, custom memory critical"
 
 setup_memcheck 90 0
 

ctdb/tests/eventscripts/05.system.monitor.017.sh

@@ -2,7 +2,7 @@
 
 . "${TEST_SCRIPTS_DIR}/unit.sh"
 
-define_test "Memory check, bad situation, both memory checks, causes unhealthy"
+define_test "Memory check, bad situation, both custom memory checks, causes unhealthy"
 
 setup_memcheck 87 0