initscript: wait until we can ping ctdbd before setting tunables.

Currently we do a "sleep 1" after starting and before running
set_ctdb_variables to set the tunables.  This is too arbitrary and
might fail if the system is heavily loaded.  This, for example, could
result in some nodes running with DeterministicIPs and some without,
in which case a different IP allocation algorithm would run depending
on who is the recmaster!

This makes the start function wait until "ctdb ping" succeeds (with 10
second timeout) before trying to run set_ctdb_variables.  If a timeout
occurs then the start function attempts to kill ctdbd before exiting
with a failure.

It also cleans up the status reporting code for Red Hat and SUSE so
that the final status code is reported.  Currently there are cases
where a correct status is prematurely reported before a failure
occurs.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit cdcd05662a30b51caaeeab4ac44138cac2474e0a)

ctdb/config/ctdb.init

@@ -164,6 +164,19 @@ set_retval() {
     return $1
 }
 
+wait_until_ready () {
+    _timeout="${1:-10}" # default is 10 seconds
+
+    _count=0
+    while ! ctdb ping >/dev/null 2>&1 ; do
+	if [ $_count -ge $_timeout ] ; then
+	    return 1
+	fi
+	sleep 1
+	_count=$(($_count + 1))
+    done
+}
+
 ctdbd=${CTDBD:-/usr/sbin/ctdbd}
 
 start() {
@@ -193,14 +206,11 @@ start() {
 	    ;;
 	suse)
 	    eval startproc $ctdbd "$CTDB_OPTIONS"
-	    rc_status -v
 	    RETVAL=$?
 	    ;;
 	redhat)
 	    eval $ctdbd "$CTDB_OPTIONS"
 	    RETVAL=$?
-	    [ $RETVAL -eq 0 ] && success || failure
-	    echo
 	    [ $RETVAL -eq 0 ] && touch /var/lock/subsys/ctdb || RETVAL=1
 	    ;;
 	debian)
@@ -210,9 +220,25 @@ start() {
 	    ;;
     esac
 
-    sleep 1
+    if [ $RETVAL -eq 0 ] ; then
+	if wait_until_ready ; then
+	    set_ctdb_variables
+	else
+	    RETVAL=1
+	    pkill -9 -f $ctdbd >/dev/null 2>&1
+	fi
+    fi
 
-    set_ctdb_variables
+    case $init_style in
+	suse)
+	    set_retval $RETVAL
+	    rc_status -v
+	    ;;
+	redhat)
+	    [ $RETVAL -eq 0 ] && success || failure
+	    echo
+	    ;;
+    esac
 
     return $RETVAL
 }