Eventscripts: "recovered" event should not fail on NATGW failure

The recovery process has no protection against the "recovered" event
failing, so this can cause a recovery loop.

Instead of failing the "recovered" event, add a "monitor" event and
fail that instead.  In this case the failure semantics are well
defined.

A separate patch should ban nodes if the "recovered" event fails for
an unknown reason.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit eaa7c165f58abd7e259c37d76b7dd37c91e13d9f)

ctdb/config/events.d/11.natgw

@@ -34,6 +34,26 @@ delete_all() {
 	iptables -D INPUT -p tcp --syn -d $_ip/32 -j REJECT 2>/dev/null
 }
 
+ensure_natgwmaster ()
+{
+    _event="$1"
+
+    set -- $(ctdb natgwlist)
+    natgwmaster="${1:--1}" # Default is -1 if natgwlist fails
+    natgwip="$2"
+
+    if [ "$natgwmaster" = "-1" ]; then
+	echo "There is no NATGW master node"
+	# The recovered event should never fail - we'll catch this
+	# failure in the monitor event.
+	if [ "$_event" = "recovered" ] ; then
+	    exit 0
+	else
+	    exit 1
+	fi
+    fi
+}
+
 case "$1" in 
     startup)
 	# Error if CTDB_NATGW_PUBLIC_IP is listed in public addresses
@@ -47,11 +67,7 @@ case "$1" in
     recovered|updatenatgw|ipreallocated)
 	mypnn=$(ctdb pnn | cut -d: -f2)
 
-	set -- $(ctdb natgwlist)
-	natgwmaster="${1:--1}" # Default is -1 if natgwlist fails
-	natgwip="$2"
-
-	[ "$natgwmaster" = "-1" ] && die "There is no NATGW master node"
+	ensure_natgwmaster "$1"
 
 	delete_all
 
@@ -83,6 +99,10 @@ case "$1" in
 	delete_all
 	;;
 
+    monitor)
+	ensure_natgwmaster "$1"
+	;;
+
     *)
 	ctdb_standard_event_handler "@"
 	;;