ctdb-scripts: Add caching function for public IPs

This is way more complicated than I would like but, as per the
comment, this is due to complexities in the way public IPs work.  The
main consumer will be statd-callout, which will then be able to run as
a non-root user.

Also generate the cache file in test code, whenever the PNN is set.
However, this can cause "ctdb ip" to generate a fake IP layout before
public IPs are setup.  So, have the "ctdb ip" stub generate the IP
layout every time it is run to avoid it being stale.

Signed-off-by: Martin Schwenke <mschwenke@ddn.com>
Reviewed-by: Volker Lendecke <vl@samba.org>

ctdb/config/events/legacy/10.interface.script

@@ -149,6 +149,8 @@ takeip)
 	ip=$3
 	maskbits=$4
 
+	update_my_public_ip_addresses "takeip" "$ip"
+
 	add_ip_to_iface "$iface" "$ip" "$maskbits" || {
 		exit 1;
 	}
@@ -177,6 +179,8 @@ releaseip)
 
 	kill_tcp_connections "$iface" "$ip"
 
+	update_my_public_ip_addresses "releaseip" "$ip"
+
 	delete_ip_from_iface "$iface" "$ip" "$maskbits" || {
 		ip_unblock "$ip" "$iface"
 		exit 1
@@ -236,6 +240,11 @@ updateip)
 	tickle_tcp_connections "$ip"
 	;;
 
+ipreallocated)
+	# Just to make sure
+	update_my_public_ip_addresses "ipreallocated"
+	;;
+
 monitor)
 	monitor_interfaces || exit 1
 	;;

ctdb/config/functions

@@ -269,6 +269,59 @@ ctdb_get_ip_address()
 	cat "$_ip_addr_file"
 }
 
+# Cache of public IP addresses assigned to this node.  This function
+# exists mainly so statd-callout does not need to talk to ctdbd, so
+# can be run as non-root, but it may be used in other places.  This
+# must be updated/refreshed on failover.  This is done in
+# 10.interface, but doing it in "ipreallocated" isn't enough because
+# clients may connect as soon as "takeip" completes.  Also, the VNN in
+# the daemon is only updated after the "releaseip" event completes, so
+# "ctdb -X ip" can't be relied on there.  Hence, complex updates
+# involving locking for "takeip" & "releaseip".  A future
+# restructuring of the failover model will obsolete all of these
+# moving parts.
+CTDB_MY_PUBLIC_IPS_CACHE="${CTDB_SCRIPT_VARDIR}/my-public-ip-addresses"
+update_my_public_ip_addresses()
+{
+	_event="$1"
+
+	_f="$CTDB_MY_PUBLIC_IPS_CACHE"
+	_lock="${_f}.lock"
+
+	# In private CTDB state directory - no $$ security issue
+	_new="${_f}.new.$$"
+	{
+		flock --timeout 10 9 ||
+			die "ctdb_get_my_public_ip_addresses: timeout"
+
+		case "$_event" in
+		takeip)
+			_ip="$2"
+			# Redirect of stderr guards against initial
+			# missing file
+			cat "$_f" 2>/dev/null >"$_new"
+			echo "$_ip" >>"$_new"
+			;;
+		releaseip)
+			_ip="$2"
+			# Redirect of stderr guards against initial
+			# missing file, which shouldn't happen in
+			# releaseip...
+			grep -Fvx "$_ip" "$_f" 2>/dev/null >"$_new"
+			;;
+		ipreallocated)
+			_pnn=$(ctdb_get_pnn)
+			$CTDB -X ip |
+				awk -F'|' -v pnn="$_pnn" \
+				    '$3 == pnn {print $2}' >"$_new"
+			;;
+		esac
+
+		mv "$_new" "$_f"
+
+	} 9>"$_lock"
+}
+
 # Cached retrieval of database options for use by event scripts.
 #
 # If the variables are already set then they should not be overwritten

ctdb/tests/UNIT/eventscripts/scripts/local.sh

@@ -318,6 +318,14 @@ ctdb_set_pnn()
 	CTDB_SCRIPT_VARDIR="${CTDB_TEST_TMP_DIR}/scripts/${FAKE_CTDB_PNN}"
 	export CTDB_SCRIPT_VARDIR
 	mkdir -p "$CTDB_SCRIPT_VARDIR"
+
+	if [ -f "${CTDB_BASE}/public_addresses" ]; then
+		ctdb ip | while read -r _ip _pnn; do
+			if [ "$_pnn" = "$FAKE_CTDB_PNN" ]; then
+				echo "$_ip"
+			fi
+		done >"${CTDB_SCRIPT_VARDIR}/my-public-ip-addresses"
+	fi
 }
 
 ctdb_get_interfaces()

ctdb/tests/UNIT/eventscripts/stubs/ctdb

@@ -195,8 +195,7 @@ ip_reallocate()
 
 ctdb_ip()
 {
-	# If nobody has done any IP-fu then generate a layout.
-	[ -f "$FAKE_CTDB_IP_LAYOUT" ] || ip_reallocate
+	ip_reallocate
 
 	_mypnn=$(ctdb_pnn)