samba-mirror/ctdb/config/functions

# Hey Emacs, this is a -*- shell-script -*- !!!

# utility functions for ctdb event scripts

if [ -z "$CTDB_BASE" ] ; then
    echo 'CTDB_BASE unset in CTDB functions file'
    exit 1
fi
export CTDB_BASE

# CTDB_VARDIR is used elsewhere
# shellcheck disable=SC2034
CTDB_VARDIR="/usr/local/var/lib/ctdb"
ctdb_rundir="/usr/local/var/run/ctdb"

CTDB="${CTDB:-/usr/local/bin/ctdb}"

# Only (and always) override these variables in test code

if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
    CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
fi

if [ -z "$CTDB_SYS_ETCDIR" ] ; then
    CTDB_SYS_ETCDIR="/etc"
fi

if [ -z "$CTDB_HELPER_BINDIR" ] ; then
    CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
fi

#######################################
# pull in a system config file, if any

rewrite_ctdb_options ()
{
    case "$CTDB_DBDIR" in
	tmpfs|tmpfs:*)
	    _opts_defaults="mode=700"
	    # Get any extra options specified after colon
	    if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
		_opts=""
	    else
		_opts="${CTDB_DBDIR#tmpfs:}"
	    fi
	    # It is OK to repeat mount options - last value wins.
	    # CTDB_DBDIR_TMPFS_OPTIONS is used by ctdbd_wrapper
	    # shellcheck disable=SC2034
	    CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"

	    CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
	    ;;
	*)
	    # shellcheck disable=SC2034
	    CTDB_DBDIR_TMPFS_OPTIONS=""
    esac
}

_loadconfig() {

    if [ -z "$1" ] ; then
	foo="${service_config:-${service_name}}"
	if [ -n "$foo" ] ; then
	    loadconfig "$foo"
	    return
	fi
    fi

    if [ "$1" != "ctdb" ] ; then
	loadconfig "ctdb"
    fi

    if [ -z "$1" ] ; then
	return
    fi

    if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/$1" ]; then
	. "${CTDB_SYS_ETCDIR}/sysconfig/$1"
    elif [ -f "${CTDB_SYS_ETCDIR}/default/$1" ]; then
	. "${CTDB_SYS_ETCDIR}/default/$1"
    elif [ -f "${CTDB_BASE}/sysconfig/$1" ]; then
	. "${CTDB_BASE}/sysconfig/$1"
    fi

    if [ "$1" = "ctdb" ] ; then
	_config="${CTDBD_CONF:-${CTDB_BASE}/ctdbd.conf}"
	if [ -r "$_config" ] ; then
	    . "$_config"
	fi
	rewrite_ctdb_options
    fi
}

loadconfig () {
    _loadconfig "$@"
}

##############################################################

# CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
# configuration file.
debug ()
{
    if [ "${CTDB_SCRIPT_DEBUGLEVEL:-2}" -ge 4 ] ; then
	# If there are arguments then echo them.  Otherwise expect to
	# use stdin, which allows us to pass lots of debug using a
	# here document.
	if [ -n "$1" ] ; then
	    echo "DEBUG: $*"
	else
	    sed -e 's@^@DEBUG: @'
	fi
    else
	if [ -z "$1" ] ; then
	    cat >/dev/null
	fi
    fi
}

die ()
{
    _msg="$1"
    _rc="${2:-1}"

    echo "$_msg" >&2
    exit "$_rc"
}

# Log given message or stdin to either syslog or a CTDB log file
# $1 is the tag passed to logger if syslog is in use.
script_log ()
{
    _tag="$1" ; shift

    case "$CTDB_LOGGING" in
	file:*|"")
	    if [ -n "$CTDB_LOGGING" ] ; then
		_file="${CTDB_LOGGING#file:}"
	    else
		_file="/usr/local/var/log/log.ctdb"
	    fi
	    {
		if [ -n "$*" ] ; then
		    echo "$*"
		else
		    cat
		fi
	    } >>"$_file"
	    ;;
	*)
	    # Handle all syslog:* variants here too.  There's no tool to do
	    # the lossy things, so just use logger.
	    logger -t "ctdbd: ${_tag}" "$*"
	    ;;
    esac
}

# When things are run in the background in an eventscript then logging
# output might get lost.  This is the "solution".  :-)
background_with_logging ()
{
    (
	"$@" 2>&1 </dev/null |
	script_log "${script_name}&"
    )&

    return 0
}

##############################################################
# check number of args for different events
ctdb_check_args ()
{
    case "$1" in
	takeip|releaseip)
	    if [ $# != 4 ]; then
		echo "ERROR: must supply interface, IP and maskbits"
		exit 1
	    fi
	    ;;
	updateip)
	    if [ $# != 5 ]; then
		echo "ERROR: must supply old interface, new interface, IP and maskbits"
		exit 1
	    fi
	    ;;
    esac
}

##############################################################
# determine on what type of system (init style) we are running
detect_init_style()
{
    # only do detection if not already set:
    [ -z "$CTDB_INIT_STYLE" ] || return

    if [ -x /sbin/startproc ]; then
        CTDB_INIT_STYLE="suse"
    elif [ -x /sbin/start-stop-daemon ]; then
        CTDB_INIT_STYLE="debian"
    else
        CTDB_INIT_STYLE="redhat"
    fi
}

######################################################
# simulate /sbin/service on platforms that don't have it
# _service() makes it easier to hook the service() function for
# testing.
_service ()
{
  _service_name="$1"
  _op="$2"

  # do nothing, when no service was specified
  [ -z "$_service_name" ] && return

  if [ -x /sbin/service ]; then
      $_nice /sbin/service "$_service_name" "$_op"
  elif [ -x /usr/sbin/service ]; then
      $_nice /usr/sbin/service "$_service_name" "$_op"
  elif [ -x /bin/systemctl ]; then
      $_nice /bin/systemctl "$_op" "$_service_name"
  elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
  elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
  fi
}

service()
{
    _nice=""
    _service "$@"
}

######################################################
# simulate /sbin/service (niced) on platforms that don't have it
nice_service()
{
    _nice="nice"
    _service "$@"
}

######################################################
# Cached retrieval of PNN from local node.  This never changes so why
# open a client connection to the server each time this is needed?
ctdb_get_pnn ()
{
    _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
    if [ ! -f "$_pnn_file" ] ; then
	$CTDB pnn | sed -e 's@.*:@@' >"$_pnn_file"
    fi

    cat "$_pnn_file"
}

# Cached retrieval of private IP address from local node.  This never
# changes.
ctdb_get_ip_address ()
{
    _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
    if [ ! -f "$_ip_addr_file" ] ; then
	$CTDB -X nodestatus |
	    awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
    fi

    # ip_address is used by caller
    # shellcheck disable=SC2034
    cat "$_ip_addr_file"
}

######################################################
# wrapper around /proc/ settings to allow them to be hooked
# for testing
# 1st arg is relative path under /proc/, 2nd arg is value to set
set_proc ()
{
    echo "$2" >"/proc/$1"
}

set_proc_maybe ()
{
    if [ -w "/proc/$1" ] ; then
	set_proc "$1" "$2"
    fi
}

######################################################
# wrapper around getting file contents from /proc/ to allow
# this to be hooked for testing
# 1st arg is relative path under /proc/
get_proc ()
{
    cat "/proc/$1"
}

######################################################
# Print up to $_max kernel stack traces for processes named $_program
program_stack_traces ()
{
    _prog="$1"
    _max="${2:-1}"

    _count=1
    for _pid in $(pidof "$_prog") ; do
	[ "$_count" -le "$_max" ] || break

	# Do this first to avoid racing with process exit
	_stack=$(get_proc "${_pid}/stack" 2>/dev/null)
	if [ -n "$_stack" ] ; then
	    echo "Stack trace for ${_prog}[${_pid}]:"
	    echo "$_stack"
	    _count=$((_count + 1))
	fi
    done
}

######################################################
# Ensure $service_name is set
assert_service_name ()
{
    [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
}

######################################################
# check a set of directories is available
# return 1 on a missing directory
# directories are read from stdin
######################################################
ctdb_check_directories_probe()
{
    while IFS="" read d ; do
	case "$d" in
	    *%*)
		continue
		;;
	    *)
		[ -d "${d}/." ] || return 1
	esac
    done
}

######################################################
# check a set of directories is available
# directories are read from stdin
######################################################
ctdb_check_directories()
{
    ctdb_check_directories_probe || {
	echo "ERROR: $service_name directory \"$d\" not available"
	exit 1
    }
}

######################################################
# check a set of tcp ports
# usage: ctdb_check_tcp_ports <ports...>
######################################################

# This flag file is created when a service is initially started.  It
# is deleted the first time TCP port checks for that service succeed.
# Until then ctdb_check_tcp_ports() prints a more subtle "error"
# message if a port check fails.
_ctdb_check_tcp_common ()
{
    assert_service_name
    _d="${CTDB_SCRIPT_VARDIR}/failcount"
    _ctdb_service_started_file="${_d}/${service_name}.started"
}

ctdb_check_tcp_init ()
{
    _ctdb_check_tcp_common
    mkdir -p "${_ctdb_service_started_file%/*}" # dirname
    touch "$_ctdb_service_started_file"
}

# Check whether something is listening on all of the given TCP ports
# using the "ctdb checktcpport" command.
ctdb_check_tcp_ports()
{
    if [ -z "$1" ] ; then
	echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
	exit 1
    fi

    for _p ; do  # process each function argument (port)
	_cmd="$CTDB checktcpport $_p"
	_out=$($_cmd 2>&1)
	_ret=$?
	case "$_ret" in
	    0)
		_ctdb_check_tcp_common
		if [ ! -f "$_ctdb_service_started_file" ] ; then
		    echo "ERROR: $service_name tcp port $_p is not responding"
		    debug "\"ctdb checktcpport $_p\" was able to bind to port"
		else
		    echo "INFO: $service_name tcp port $_p is not responding"
		fi

		return 1
		;;
	    98)
		# Couldn't bind, something already listening, next port...
		continue
		;;
	    *)
		echo "ERROR: unexpected error running \"ctdb checktcpport\""
		debug <<EOF
$CTDB checktcpport (exited with $_ret) with output:
$_out"
EOF
		return $_ret
	esac
    done

    # All ports listening
    _ctdb_check_tcp_common
    rm -f "$_ctdb_service_started_file"
    return 0
}

######################################################
# check a unix socket
# usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
######################################################
ctdb_check_unix_socket() {
    socket_path="$1"
    [ -z "$socket_path" ] && return

    if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
        echo "ERROR: $service_name socket $socket_path not found"
        return 1
    fi
}

######################################################
# check a command returns zero status
# usage: ctdb_check_command <command>
######################################################
ctdb_check_command ()
{
    _out=$("$@" 2>&1) || {
	echo "ERROR: $* returned error"
	echo "$_out" | debug
	exit 1
    }
}

################################################
# kill off any TCP connections with the given IP
################################################
kill_tcp_connections ()
{
    _iface="$1"
    _ip="$2"

    _oneway=false
    if [ "$3" = "oneway" ] ; then
	_oneway=true
    fi

    get_tcp_connections_for_ip "$_ip" | {
	_killcount=0
	_connections=""
	_nl="
"
	while read _dst _src; do
	    _destport="${_dst##*:}"
	    __oneway=$_oneway
	    case $_destport in
		# we only do one-way killtcp for CIFS
		139|445) __oneway=true ;;
	    esac

	    echo "Killing TCP connection $_src $_dst"
	    _connections="${_connections}${_nl}${_src} ${_dst}"
	    if ! $__oneway ; then
		_connections="${_connections}${_nl}${_dst} ${_src}"
	    fi

	    _killcount=$((_killcount + 1))
	done

	if [ $_killcount -eq 0 ] ; then
	    return
	fi

	echo "$_connections" | \
		"${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
		echo "Failed to kill TCP connections"
		return
	}

	_remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l)

	if [ "$_remaining" -eq 0 ] ; then
		echo "Killed $_killcount TCP connections to released IP $_ip"
		return
	fi

	_t="${_remaining}/${_killcount}"
	echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)"
    }
}

##################################################################
# kill off the local end for any TCP connections with the given IP
##################################################################
kill_tcp_connections_local_only ()
{
    kill_tcp_connections "$@" "oneway"
}

##################################################################
# tickle any TCP connections with the given IP
##################################################################
tickle_tcp_connections ()
{
    _ip="$1"

    get_tcp_connections_for_ip "$_ip" |
    {
	_failed=false

	while read dest src; do
	    echo "Tickle TCP connection $src $dest"
	    $CTDB tickle "$src" "$dest" >/dev/null 2>&1 || _failed=true
	    echo "Tickle TCP connection $dest $src"
	    $CTDB tickle "$dest" "$src" >/dev/null 2>&1 || _failed=true
	done

	if $_failed ; then
	    echo "Failed to send tickle control"
	fi
    }
}

get_tcp_connections_for_ip ()
{
    _ip="$1"

    ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
}

########################################################

add_ip_to_iface ()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # Ensure interface is up
    ip link set "$_iface" up || \
	die "Failed to bringup interface $_iface"

    # Only need to define broadcast for IPv4
    case "$_ip" in
        *:*) _bcast=""      ;;
	*)   _bcast="brd +" ;;
    esac

    # Intentionally unquoted multi-word value here
    # shellcheck disable=SC2086
    ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
	echo "Failed to add $_ip/$_maskbits on dev $_iface"
	return 1
    }

    # Wait 5 seconds for IPv6 addresses to stop being tentative...
    if [ -z "$_bcast" ] ; then
	for _x in $(seq 1 10) ; do
	    ip addr show to "${_ip}/128" | grep -q "tentative" || break
	    sleep 0.5
	done

	# If the address was a duplicate then it won't be on the
	# interface so flag an error.
	_t=$(ip addr show to "${_ip}/128")
	case "$_t" in
	    "")
		echo "Failed to add $_ip/$_maskbits on dev $_iface"
		return 1
		;;
	    *tentative*|*dadfailed*)
		echo "Failed to add $_ip/$_maskbits on dev $_iface"
		ip addr del "$_ip/$_maskbits" dev "$_iface"
		return 1
		;;
	esac
    fi
}

delete_ip_from_iface()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # This could be set globally for all interfaces but it is probably
    # better to avoid surprises, so limit it the interfaces where CTDB
    # has public IP addresses.  There isn't anywhere else convenient
    # to do this so just set it each time.  This is much cheaper than
    # remembering and re-adding secondaries.
    set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1

    ip addr del "$_ip/$_maskbits" dev "$_iface" || {
	echo "Failed to del $_ip on dev $_iface"
	return 1
    }
}

# If the given IP is hosted then print 2 items: maskbits and iface
ip_maskbits_iface ()
{
    _addr="$1"

    case "$_addr" in
	*:*) _bits=128 ;;
	*)   _bits=32  ;;
    esac

    ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
	awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
		       sub("@.*", "", iface) } \
             $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
                           print mask, iface }'
}

drop_ip ()
{
    _addr="${1%/*}"  # Remove optional maskbits

    # Intentional word splitting here
    # shellcheck disable=SC2046
    set -- $(ip_maskbits_iface "$_addr")
    if [ -n "$1" ] ; then
	_maskbits="$1"
	_iface="$2"
	echo "Removing public address $_addr/$_maskbits from device $_iface"
	delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
    fi
}

drop_all_public_ips ()
{
	# _x is intentionally ignored
	# shellcheck disable=SC2034
	while read _ip _x ; do
		drop_ip "$_ip"
	done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
}

flush_route_cache ()
{
    set_proc_maybe sys/net/ipv4/route/flush 1
    set_proc_maybe sys/net/ipv6/route/flush 1
}

########################################################
# Interface monitoring

# If the interface is a virtual one (e.g. VLAN) then get the
# underlying interface
interface_get_real ()
{
    # Output of "ip link show <iface>"
    _iface_info="$1"

    # Extract the full interface description to see if it is a VLAN
    _t=$(echo "$_iface_info" |
		awk 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
			       print iface }')
    case "$_t" in
	*@*)
	    # VLAN: use the underlying interface, after the '@'
	    echo "${_t##*@}"
	    ;;
	*)
	    # Not a regular VLAN.  For backward compatibility, assume
	    # there is some other sort of VLAN that doesn't have the
	    # '@' in the output and only use what is before a '.'.  If
	    # there is no '.' then this will be the whole interface
	    # name.
	    echo "${_t%%.*}"
    esac
}

# Check whether an interface is operational
interface_monitor ()
{
    _iface="$1"

    _iface_info=$(ip link show "$_iface" 2>&1) || {
	echo "ERROR: Monitored interface ${_iface} does not exist"
	return 1
    }


    # If the interface is a virtual one (e.g. VLAN) then get the
    # underlying interface.
    _realiface=$(interface_get_real "$_iface_info")

    if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null) ; then
	# This is a bond: various monitoring strategies
	echo "$_bi" | grep -q 'Currently Active Slave: None' && {
	    echo "ERROR: No active slaves for bond device ${_realiface}"
	    return 1
	}
	echo "$_bi" | grep -q '^MII Status: up' || {
	    echo "ERROR: public network interface ${_realiface} is down"
	    return 1
	}
	echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
	    # This works around a bug in the driver where the
	    # overall bond status can be up but none of the actual
	    # physical interfaces have a link.
	    echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
		echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
		return 1
	    }
	}

	return 0
    else
	# Not a bond
	case "$_iface" in
	    lo*)
		# loopback is always working
		return 0
		;;
	    ib*)
		# we don't know how to test ib links
		return 0
		;;
	    *)
		ethtool "$_iface" | grep -q 'Link detected: yes' || {
		    # On some systems, this is not successful when a
		    # cable is plugged but the interface has not been
		    # brought up previously. Bring the interface up
		    # and try again...
		    ip link set "$_iface" up
		    ethtool "$_iface" | grep -q 'Link detected: yes' || {
			echo "ERROR: No link on the public network interface ${_iface}"
			return 1
		    }
		}
		return 0
		;;
	esac
    fi
}

########################################################
# Simple counters
_ctdb_counter_common () {
    _service_name="${1:-${service_name:-${script_name}}}"
    _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
    mkdir -p "${_counter_file%/*}" # dirname
}
ctdb_counter_init () {
    _ctdb_counter_common "$1"

    >"$_counter_file"
}
ctdb_counter_incr () {
    _ctdb_counter_common "$1"

    # unary counting!
    echo -n 1 >> "$_counter_file"
}
ctdb_counter_get () {
    _ctdb_counter_common "$1"
    # unary counting!
    stat -c "%s" "$_counter_file" 2>/dev/null || echo 0
}

########################################################

ctdb_setup_service_state_dir ()
{
	_s="${1:-${service_name}}"

	_service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${_s}"
	mkdir -p "$_service_state_dir" ||
		die "Error creating state dir \"${_service_state_dir}\""

	echo "$_service_state_dir"
}

########################################################
# Managed status history, for auto-start/stop

_ctdb_managed_common ()
{
    _ctdb_managed_file="${CTDB_SCRIPT_VARDIR}/managed_history/${service_name}"
}

ctdb_service_managed ()
{
    _ctdb_managed_common
    mkdir -p "${_ctdb_managed_file%/*}" # dirname
    touch "$_ctdb_managed_file"
}

ctdb_service_unmanaged ()
{
    _ctdb_managed_common
    rm -f "$_ctdb_managed_file"
}

is_ctdb_previously_managed_service ()
{
    _ctdb_managed_common
    [ -f "$_ctdb_managed_file" ]
}

##################################################################
# Reconfigure a service on demand

_ctdb_service_reconfigure_common ()
{
    _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
    mkdir -p "$_d"
    _ctdb_service_reconfigure_flag="$_d/reconfigure"
}

ctdb_service_needs_reconfigure ()
{
    _ctdb_service_reconfigure_common
    [ -e "$_ctdb_service_reconfigure_flag" ]
}

ctdb_service_set_reconfigure ()
{
    _ctdb_service_reconfigure_common
    >"$_ctdb_service_reconfigure_flag"
}

ctdb_service_unset_reconfigure ()
{
    _ctdb_service_reconfigure_common
    rm -f "$_ctdb_service_reconfigure_flag"
}

ctdb_service_reconfigure ()
{
    echo "Reconfiguring service \"${service_name}\"..."
    ctdb_service_unset_reconfigure
    service_reconfigure || return $?
    ctdb_counter_init
}

# Default service_reconfigure() function does nothing.
service_reconfigure ()
{
    :
}

ctdb_reconfigure_take_lock ()
{
    _ctdb_service_reconfigure_common
    _lock="${_d}/reconfigure_lock"
    mkdir -p "${_lock%/*}" # dirname
    touch "$_lock"

    (
	flock 0
	# This is overkill but will work if we need to extend this to
	# allow certain events to run multiple times in parallel
	# (e.g. takeip) and write multiple PIDs to the file.
	read _locker_event 
	if [ -n "$_locker_event" ] ; then
	    while read _pid ; do
		if [ -n "$_pid" -a "$_pid" != $$ ] && \
		    kill -0 "$_pid" 2>/dev/null ; then
		    exit 1
		fi
	    done
	fi

	printf "%s\n%s\n" "$event_name" $$ >"$_lock"
	exit 0
    ) <"$_lock"
}

ctdb_reconfigure_release_lock ()
{
    _ctdb_service_reconfigure_common
    _lock="${_d}/reconfigure_lock"

    rm -f "$_lock"
}

ctdb_replay_monitor_status ()
{
    echo "Replaying previous status for this script due to reconfigure..."
    # Leading separator ('|') is missing in some versions...
    _out=$($CTDB scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
    # Output looks like this:
    # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
    # This is the cheapest way of getting fields in the middle.
    # Intentional word splitting here
    # shellcheck disable=SC2046,2086
    set -- $(IFS="|" ; echo $_out)
    _code="$3"
    _status="$4"
    # The error output field can include colons so we'll try to
    # preserve them.  The weak checking at the beginning tries to make
    # this work for both broken (no leading '|') and fixed output.
    _out="${_out%|}"
    _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
    case "$_status" in
	OK) : ;;  # Do nothing special.
	TIMEDOUT)
	    # Recast this as an error, since we can't exit with the
	    # correct negative number.
	    _code=1
	    _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
	    ;;
	DISABLED)
	    # Recast this as an OK, since we can't exit with the
	    # correct negative number.
	    _code=0
	    _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
	    ;;
	*) : ;;  # Must be ERROR, do nothing special.
    esac
    if [ -n "$_err_out" ] ; then
	echo "$_err_out"
    fi
    exit $_code
}

ctdb_service_check_reconfigure ()
{
    assert_service_name

    # We only care about some events in this function.  For others we
    # return now.
    case "$event_name" in
	monitor|ipreallocated|reconfigure) : ;;
	*) return 0 ;;
    esac

    if ctdb_reconfigure_take_lock ; then
	# No events covered by this function are running, so proceed
	# with gay abandon.
	case "$event_name" in
	    reconfigure)
		(ctdb_service_reconfigure)
		exit $?
		;;
	    ipreallocated)
		if ctdb_service_needs_reconfigure ; then
		    ctdb_service_reconfigure
		fi
		;;
	esac

	ctdb_reconfigure_release_lock
    else
	# Somebody else is running an event we don't want to collide
	# with.  We proceed with caution.
	case "$event_name" in
	    reconfigure)
		# Tell whoever called us to retry.
		exit 2
		;;
	    ipreallocated)
		# Defer any scheduled reconfigure and just run the
		# rest of the ipreallocated event, as per the
		# eventscript.  There's an assumption here that the
		# event doesn't depend on any scheduled reconfigure.
		# This is true in the current code.
		return 0
		;;
	    monitor)
		# There is most likely a reconfigure in progress so
		# the service is possibly unstable.  As above, we
		# defer any scheduled reconfigured.  We also replay
		# the previous monitor status since that's the best
		# information we have.
		ctdb_replay_monitor_status
		;;
	esac
    fi
}

##################################################################
# Does CTDB manage this service? - and associated auto-start/stop

ctdb_compat_managed_service ()
{
    if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
	CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
    fi
}

is_ctdb_managed_service ()
{
    assert_service_name

    # $t is used just for readability and to allow better accurate
    # matching via leading/trailing spaces
    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    if [ "${t#* ${service_name} }" != "${t}" ] ; then
	return 0
    fi

    # If above didn't match then update $CTDB_MANAGED_SERVICES for
    # backward compatibility and try again.
    ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
    ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
    ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
    ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"

    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    [ "${t#* ${service_name} }" != "${t}" ]
}

ctdb_start_stop_service ()
{
    assert_service_name

    # Allow service-start/service-stop pseudo-events to start/stop
    # services when we're not auto-starting/stopping and we're not
    # monitoring.
    case "$event_name" in
	service-start)
	    if is_ctdb_managed_service ; then
		die 'service-start event not permitted when service is managed'
	    fi
	    if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
		die 'service-start event not permitted with CTDB_SERVICE_AUTOSTARTSTOP=yes'
	    fi
	    ctdb_service_start
	    exit $?
	    ;;
	service-stop)
	    if is_ctdb_managed_service ; then
		die 'service-stop event not permitted when service is managed'
	    fi
	    if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
		die 'service-stop event not permitted with CTDB_SERVICE_AUTOSTARTSTOP=yes'
	    fi
	    ctdb_service_stop
	    exit $?
	    ;;
    esac

    # Do nothing unless configured to...
    [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0

    [ "$event_name" = "monitor" ] || return 0

    if is_ctdb_managed_service ; then
	if ! is_ctdb_previously_managed_service ; then
	    echo "Starting service \"$service_name\" - now managed"
	    background_with_logging ctdb_service_start
	    exit $?
	fi
    else
	if is_ctdb_previously_managed_service ; then
	    echo "Stopping service \"$service_name\" - no longer managed"
	    background_with_logging ctdb_service_stop
	    exit $?
	fi
    fi
}

ctdb_service_start ()
{
    # The service is marked managed if we've ever tried to start it.
    ctdb_service_managed

    service_start || return $?

    ctdb_counter_init
    ctdb_check_tcp_init
}

ctdb_service_stop ()
{
    ctdb_service_unmanaged
    service_stop
}

# Default service_start() and service_stop() functions.
 
# These may be overridden in an eventscript.
service_start ()
{
    service "$service_name" start
}

service_stop ()
{
    service "$service_name" stop
}

##################################################################

# This exists only for backward compatibility with 3rd party scripts
# that call it
ctdb_standard_event_handler ()
{
    :
}

iptables_wrapper ()
{
    _family="$1" ; shift
    if [ "$_family" = "inet6" ] ; then
	_iptables_cmd="ip6tables"
    else
	_iptables_cmd="iptables"
    fi

    # iptables doesn't like being re-entered, so flock-wrap it.
    flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
}

# AIX (and perhaps others?) doesn't have mktemp
if ! type mktemp >/dev/null 2>&1 ; then
    mktemp ()
    {
	_dir=false
	if [ "$1" = "-d" ] ; then
	    _dir=true
	    shift
	fi
	_d="${TMPDIR:-/tmp}"
	_hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
	    md5sum | \
	    sed -e 's@\(..........\).*@\1@')
	_t="${_d}/tmp.${_hex10}"
	(
	    umask 077
	    if $_dir ; then
		mkdir "$_t"
	    else
		>"$_t"
	    fi
	)
	echo "$_t"
    }
fi

######################################################################
# NFS callout handling

nfs_callout_init ()
{
	_state_dir="$1"

	if [ -z "$CTDB_NFS_CALLOUT" ] ; then
		CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
	fi
	# Always export, for statd callout
	export CTDB_NFS_CALLOUT

	# If the callout wants to use this then it must create it
	export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"

	# Export, if set, for use by clustered NFS callouts
	if [ -n "$CTDB_NFS_STATE_FS_TYPE" ] ; then
		export CTDB_NFS_STATE_FS_TYPE
	fi
	if [ -n "$CTDB_NFS_STATE_MNT" ] ; then
		export CTDB_NFS_STATE_MNT
	fi

	nfs_callout_cache="${_state_dir}/nfs_callout_cache"
	nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
	nfs_callout_cache_ops="${nfs_callout_cache}/ops"
}

nfs_callout_register ()
{
    mkdir -p "$nfs_callout_cache_ops"
    rm -f "$nfs_callout_cache_ops"/*

    echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"

    _t=$(eval "$CTDB_NFS_CALLOUT" "register")
    if [ -n "$_t" ] ; then
	echo "$_t" |
	    while IFS="" read _op ; do
		touch "${nfs_callout_cache_ops}/${_op}"
	    done
    else
	touch "${nfs_callout_cache_ops}/ALL"
    fi
}

nfs_callout ()
{
    # Re-run registration if $CTDB_NFS_CALLOUT has changed
    _prev=""
    if [ -r "$nfs_callout_cache_callout" ] ; then
	read _prev <"$nfs_callout_cache_callout"
    fi
    if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
	nfs_callout_register
    fi

    # Run the operation if it is registered...
    if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
	   [ -e "${nfs_callout_cache_ops}/ALL" ]; then
	eval "$CTDB_NFS_CALLOUT" "$@"
    fi
}

########################################################
# tickle handling
########################################################

update_tickles ()
{
	_port="$1"

	tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
	mkdir -p "$tickledir"

	# What public IPs do I hold?
	_pnn=$(ctdb_get_pnn)
	_ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')

	# IPs and port as ss filters
	_ip_filter=""
	for _ip in $_ips ; do
	    _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
	done
	_port_filter="sport == :${_port}"

	# Record connections to our public IPs in a temporary file.
	# This temporary file is in CTDB's private state directory and
	# $$ is used to avoid a very rare race involving CTDB's script
	# debugging.  No security issue, nothing to see here...
	_my_connections="${tickledir}/${_port}.connections.$$"
	# Parentheses are needed around the filters for precedence but
	# the parentheses can't be empty!
	ss -tn state established \
	   "${_ip_filter:+( ${_ip_filter} )}" \
	   "${_port_filter:+( ${_port_filter} )}" |
	awk 'NR > 1 {print $4, $3}' |
	sort >"$_my_connections"

	# Record our current tickles in a temporary file
	_my_tickles="${tickledir}/${_port}.tickles.$$"
	for _i in $_ips ; do
		$CTDB -X gettickles "$_i" "$_port" |
		awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
	done |
	sort >"$_my_tickles"

	# Add tickles for connections that we haven't already got tickles for
	comm -23 "$_my_connections" "$_my_tickles" |
	while read _src _dst ; do
		$CTDB addtickle "$_src" "$_dst"
	done

	# Remove tickles for connections that are no longer there
	comm -13 "$_my_connections" "$_my_tickles" |
	while read _src _dst ; do
		$CTDB deltickle "$_src" "$_dst"
	done

	rm -f "$_my_connections" "$_my_tickles"

	# Remove stale files from killed scripts
	find "$tickledir" -type f -mmin +10 | xargs -r rm
}

########################################################
# load a site local config file
########################################################

[ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
	. "$CTDB_RC_LOCAL"
}

[ -x "${CTDB_BASE}/rc.local" ] && {
	. "${CTDB_BASE}/rc.local"
}

[ -d "${CTDB_BASE}/rc.local.d" ] && {
	for i in "${CTDB_BASE}/rc.local.d"/* ; do
		[ -x "$i" ] && . "$i"
	done
}

script_name="${0##*/}"       # basename
event_name="$1"