onnode: add "any" nodespec to select any node with running CTDB.

In testing and other situations (e.g. eventscripts) it is necessary to
select a node where a ctdb command can be run.  The whole idea here is
to avoid nodes where ctdbd is not running and where most ctdb commands
would fail.  This implements a standard way of doing this involving a
recursive onnode command.

There is still a small window for a race, where the selected node is
suddenly shutdown, but this is unavoidable.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit fb47cce86c0edae5caaf485f13ae7a151b6cb00d)

ctdb/tools/onnode

@@ -35,7 +35,7 @@ Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
     -p          Run command in parallel on specified nodes.
     -q          Do not print node addresses (overrides -v).
     -v          Print node address even for a single node.
-  <NODES>       "all", "ok" (or "healthy"), "con" (or "connected"),
+  <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected"),
                 "rm" (or "recmaster"), "lvs" (or "lvsmaster"),
                 "natgw" (or "natgwlist");
                 or a node number (0 base); or
@@ -112,7 +112,7 @@ parse_nodespec ()
 	    case "$i" in
 		*-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
 		# Separate lines for readability.
-		all|ok|healthy|con|connected) echo "$i" ;;
+		all|any|ok|healthy|con|connected) echo "$i" ;;
 		rm|recmaster|lvs|lvsmaster|natgw|natgwlist) echo "$i" ;;
 		*)
 		    [ $i -gt -1 ] 2>/dev/null || invalid_nodespec
@@ -198,6 +198,24 @@ get_node_with_property ()
     fi
 }
 
+get_any_available_node ()
+{
+    local all_nodes="$1"
+
+    # We do a recursive onnode to find which nodes are up and running.
+    local out=$($0 -pq all ctdb pnn 2>&1)
+    local line
+    while read line ; do 
+	local pnn="${line#PNN:}"
+	if [ "$pnn" != "$line" ] ; then
+	    echo_nth "$pnn" $all_nodes
+	    return 0
+	fi
+	# Else must be an error message from a down node.
+    done <<<"$out"
+    return 1
+}
+
 get_nodes ()
 {
     local all_nodes
@@ -217,6 +235,9 @@ get_nodes ()
 	    all)
 		echo "${all_nodes//#DEAD/}"
 		;;
+	    any)
+		get_any_available_node "$all_nodes" || exit 1
+		;;
 	    ok|healthy) 
 		get_nodes_with_status "$all_nodes" "healthy" || exit 1
 		;;