mirror of
https://github.com/samba-team/samba.git
synced 2024-12-22 13:34:15 +03:00
4569c65288
This allows CTDB to be configured to use "ss -K" to reset TCP connections on "releaseip". This is only supported when the kernel is configured with CONFIG_INET_DIAG_DESTROY enabled. From the documentation: ss -K has been supported in ss since iproute 4.5 in March 2016 and in the Linux kernel since 4.4 in December 2015. However, the required kernel configuration item CONFIG_INET_DIAG_DESTROY is disabled by default. Although enabled in Debian kernels since ~2017 and in Ubuntu since at least 18.04,, this has only recently been enabled in distributions such as RHEL. There seems to be no way, including running ss -K, to determine if this is supported, so use of this feature needs to be configurable. When available, it should be the fastest, most reliable way of killing connections. For RHEL and derivatives, this was enabled as follows: * RHEL 8 via https://bugzilla.redhat.com/show_bug.cgi?id=2230213, arriving in version kernel-4.18.0-513.5.1.el8_9 * RHEL 9 via https://issues.redhat.com/browse/RHEL-212, arriving in kernel-5.14.0-360.el9 Enabling this option results in a small behaviour change because ss -K always does a 2-way kill (i.e. it also sends a RST to the client). Only a 1-way kill is done for SMB connections when ctdb_killtcp is used - the reasons for this are shrouded in history and the 2-way kill seems to work fine. For the summary that is logged, when CTDB_KILLTCP_USE_SS_KILL is "yes" or "try", always log the method used, even the fallback to ctdb_killtcp. However, when set to "no", maintain the existing output. The decision to use -K rather than --kill is because short options are trivial to implement in test stubs. Signed-off-by: Martin Schwenke <mschwenke@ddn.com> Reviewed-by: Volker Lendecke <vl@samba.org> Reviewed-by: Jerry Heyman <jheyman@ddn.com> Autobuild-User(master): Martin Schwenke <martins@samba.org> Autobuild-Date(master): Thu Nov 7 00:12:34 UTC 2024 on atb-devel-224
1284 lines
29 KiB
Bash
Executable File
1284 lines
29 KiB
Bash
Executable File
# Hey Emacs, this is a -*- shell-script -*- !!!
|
|
|
|
# utility functions for ctdb event scripts
|
|
|
|
if [ -z "$CTDB_BASE" ]; then
|
|
echo 'CTDB_BASE unset in CTDB functions file'
|
|
exit 1
|
|
fi
|
|
export CTDB_BASE
|
|
|
|
# CTDB_VARDIR is used elsewhere
|
|
# shellcheck disable=SC2034
|
|
CTDB_VARDIR="/usr/local/var/lib/ctdb"
|
|
|
|
CTDB="${CTDB:-/usr/local/bin/ctdb}"
|
|
|
|
# Only (and always) override these variables in test code
|
|
|
|
if [ -z "$CTDB_SCRIPT_VARDIR" ]; then
|
|
CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/scripts"
|
|
fi
|
|
|
|
if [ -z "$CTDB_SYS_ETCDIR" ]; then
|
|
CTDB_SYS_ETCDIR="/etc"
|
|
fi
|
|
|
|
if [ -z "$CTDB_HELPER_BINDIR" ]; then
|
|
CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
|
|
fi
|
|
|
|
#######################################
|
|
# pull in a system config file, if any
|
|
|
|
load_system_config()
|
|
{
|
|
for _i; do
|
|
|
|
if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/${_i}" ]; then
|
|
. "${CTDB_SYS_ETCDIR}/sysconfig/${_i}"
|
|
return
|
|
elif [ -f "${CTDB_SYS_ETCDIR}/default/${_i}" ]; then
|
|
. "${CTDB_SYS_ETCDIR}/default/${_i}"
|
|
return
|
|
fi
|
|
done
|
|
}
|
|
|
|
# load_script_options [ component script ]
|
|
# script is an event script name relative to a component
|
|
# component is currently ignored
|
|
load_script_options()
|
|
{
|
|
if [ $# -eq 2 ]; then
|
|
_script="$2"
|
|
elif [ $# -eq 0 ]; then
|
|
_script=""
|
|
else
|
|
die "usage: load_script_options [ component script ]"
|
|
fi
|
|
|
|
_options="${CTDB_BASE}/script.options"
|
|
|
|
if [ -r "$_options" ]; then
|
|
. "$_options"
|
|
fi
|
|
|
|
if [ -n "$_script" ]; then
|
|
_s="${CTDB_BASE}/events/legacy/${_script}"
|
|
else
|
|
_s="${0%.script}"
|
|
fi
|
|
_options="${_s}.options"
|
|
|
|
if [ -r "$_options" ]; then
|
|
. "$_options"
|
|
fi
|
|
}
|
|
|
|
##############################################################
|
|
|
|
die()
|
|
{
|
|
_msg="$1"
|
|
_rc="${2:-1}"
|
|
|
|
echo "$_msg" >&2
|
|
exit "$_rc"
|
|
}
|
|
|
|
# Log given message or stdin to either syslog or a CTDB log file
|
|
# $1 is the tag passed to logger if syslog is in use.
|
|
script_log()
|
|
{
|
|
_tag="$1"
|
|
shift
|
|
|
|
case "$CTDB_LOGGING" in
|
|
file:)
|
|
if [ -n "$*" ] ; then
|
|
echo "$*"
|
|
else
|
|
cat
|
|
fi >&2
|
|
;;
|
|
file:* | "")
|
|
if [ -n "$CTDB_LOGGING" ]; then
|
|
_file="${CTDB_LOGGING#file:}"
|
|
else
|
|
_file="/usr/local/var/log/log.ctdb"
|
|
fi
|
|
{
|
|
if [ -n "$*" ]; then
|
|
echo "$*"
|
|
else
|
|
cat
|
|
fi
|
|
} >>"$_file"
|
|
;;
|
|
*)
|
|
# Handle all syslog:* variants here too. There's no tool to do
|
|
# the lossy things, so just use logger.
|
|
logger -t "ctdbd: ${_tag}" "$@"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# When things are run in the background in an eventscript then logging
|
|
# output might get lost. This is the "solution". :-)
|
|
background_with_logging()
|
|
{
|
|
(
|
|
"$@" 2>&1 </dev/null |
|
|
script_log "${script_name}&"
|
|
) &
|
|
|
|
return 0
|
|
}
|
|
|
|
##############################################################
|
|
# check number of args for different events
|
|
ctdb_check_args()
|
|
{
|
|
case "$1" in
|
|
takeip | releaseip)
|
|
if [ $# != 4 ]; then
|
|
echo "ERROR: must supply interface, IP and maskbits"
|
|
exit 1
|
|
fi
|
|
;;
|
|
updateip)
|
|
if [ $# != 5 ]; then
|
|
echo "ERROR: must supply old interface, new interface, IP and maskbits"
|
|
exit 1
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
##############################################################
|
|
# determine on what type of system (init style) we are running
|
|
detect_init_style()
|
|
{
|
|
_init_style_file="${CTDB_SCRIPT_VARDIR}/init-style"
|
|
|
|
if [ ! -f "$_init_style_file" ]; then
|
|
if [ -n "$CTDB_INIT_STYLE" ]; then
|
|
echo "$CTDB_INIT_STYLE" >"$_init_style_file"
|
|
return
|
|
fi
|
|
|
|
# Subshell to contain variables in os-release file
|
|
(
|
|
_os_release="${CTDB_SYS_ETCDIR}/os-release"
|
|
if [ -f "$_os_release" ]; then
|
|
. "$_os_release"
|
|
case "$ID" in
|
|
centos | fedora | rhel)
|
|
echo "redhat"
|
|
;;
|
|
debian | ubuntu)
|
|
echo "debian"
|
|
;;
|
|
sles | suse)
|
|
echo "suse"
|
|
;;
|
|
*)
|
|
case "$ID_LIKE" in
|
|
*centos* | *rhel*)
|
|
echo "redhat"
|
|
;;
|
|
*)
|
|
echo "$ID"
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
else
|
|
echo "WARNING: unknown distribution ${ID}" >&2
|
|
echo "unknown"
|
|
fi
|
|
) >"$_init_style_file"
|
|
fi
|
|
|
|
read -r CTDB_INIT_STYLE <"$_init_style_file"
|
|
}
|
|
|
|
######################################################
|
|
# simulate /sbin/service on platforms that don't have it
|
|
# _service() makes it easier to hook the service() function for
|
|
# testing.
|
|
_service()
|
|
{
|
|
_service_name="$1"
|
|
_op="$2"
|
|
|
|
# do nothing, when no service was specified
|
|
[ -z "$_service_name" ] && return
|
|
|
|
if [ -x /sbin/service ]; then
|
|
$_nice /sbin/service "$_service_name" "$_op"
|
|
elif [ -x /usr/sbin/service ]; then
|
|
$_nice /usr/sbin/service "$_service_name" "$_op"
|
|
elif [ -x /bin/systemctl ]; then
|
|
$_nice /bin/systemctl "$_op" "$_service_name"
|
|
elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
|
|
$_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
|
|
elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
|
|
$_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
|
|
fi
|
|
}
|
|
|
|
service()
|
|
{
|
|
_nice=""
|
|
_service "$@"
|
|
}
|
|
|
|
######################################################
|
|
# simulate /sbin/service (niced) on platforms that don't have it
|
|
nice_service()
|
|
{
|
|
_nice="nice"
|
|
_service "$@"
|
|
}
|
|
|
|
######################################################
|
|
# Cached retrieval of PNN from local node. This never changes so why
|
|
# open a client connection to the server each time this is needed?
|
|
ctdb_get_pnn()
|
|
{
|
|
_pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
|
|
if [ ! -f "$_pnn_file" ]; then
|
|
$CTDB pnn >"$_pnn_file"
|
|
fi
|
|
|
|
cat "$_pnn_file"
|
|
}
|
|
|
|
# Cached retrieval of private IP address from local node. This never
|
|
# changes.
|
|
ctdb_get_ip_address()
|
|
{
|
|
_ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
|
|
if [ ! -f "$_ip_addr_file" ]; then
|
|
$CTDB -X nodestatus |
|
|
awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
|
|
fi
|
|
|
|
cat "$_ip_addr_file"
|
|
}
|
|
|
|
# Cache of public IP addresses assigned to this node. This function
|
|
# exists mainly so statd_callout does not need to talk to ctdbd, so
|
|
# can be run as non-root, but it may be used in other places. This
|
|
# must be updated/refreshed on failover. This is done in
|
|
# 10.interface, but doing it in "ipreallocated" isn't enough because
|
|
# clients may connect as soon as "takeip" completes. Also, the VNN in
|
|
# the daemon is only updated after the "releaseip" event completes, so
|
|
# "ctdb -X ip" can't be relied on there. Hence, complex updates
|
|
# involving locking for "takeip" & "releaseip". A future
|
|
# restructuring of the failover model will obsolete all of these
|
|
# moving parts.
|
|
CTDB_MY_PUBLIC_IPS_CACHE="${CTDB_SCRIPT_VARDIR}/my-public-ip-addresses"
|
|
update_my_public_ip_addresses()
|
|
{
|
|
_event="$1"
|
|
|
|
_f="$CTDB_MY_PUBLIC_IPS_CACHE"
|
|
_lock="${_f}.lock"
|
|
|
|
# In private CTDB state directory - no $$ security issue
|
|
_new="${_f}.new.$$"
|
|
{
|
|
flock --timeout 10 9 ||
|
|
die "ctdb_get_my_public_ip_addresses: timeout"
|
|
|
|
case "$_event" in
|
|
takeip)
|
|
_ip="$2"
|
|
# Redirect of stderr guards against initial
|
|
# missing file
|
|
cat "$_f" 2>/dev/null >"$_new"
|
|
echo "$_ip" >>"$_new"
|
|
;;
|
|
releaseip)
|
|
_ip="$2"
|
|
# Redirect of stderr guards against initial
|
|
# missing file, which shouldn't happen in
|
|
# releaseip...
|
|
grep -Fvx "$_ip" "$_f" 2>/dev/null >"$_new"
|
|
;;
|
|
ipreallocated)
|
|
_pnn=$(ctdb_get_pnn)
|
|
$CTDB -X ip |
|
|
awk -F'|' -v pnn="$_pnn" \
|
|
'$3 == pnn {print $2}' >"$_new"
|
|
;;
|
|
esac
|
|
|
|
mv "$_new" "$_f"
|
|
|
|
} 9>"$_lock"
|
|
}
|
|
|
|
# Cached retrieval of database options for use by event scripts.
|
|
#
|
|
# If the variables are already set then they should not be overwritten
|
|
# - this should only happen during event script testing.
|
|
ctdb_get_db_options()
|
|
{
|
|
_db_opts_file="${CTDB_SCRIPT_VARDIR}/db_options.cache"
|
|
|
|
if [ ! -f "$_db_opts_file" ]; then
|
|
{
|
|
ctdb_translate_option "database" \
|
|
"volatile database directory" \
|
|
"CTDB_DBDIR"
|
|
ctdb_translate_option "database" \
|
|
"persistent database directory" \
|
|
"CTDB_DBDIR_PERSISTENT"
|
|
ctdb_translate_option "database" \
|
|
"state database directory" \
|
|
"CTDB_DBDIR_STATE"
|
|
} >"$_db_opts_file"
|
|
fi
|
|
|
|
. "$_db_opts_file"
|
|
}
|
|
|
|
ctdb_translate_option()
|
|
{
|
|
_section="$1"
|
|
_opt="$2"
|
|
_variable="$3"
|
|
|
|
# ctdb-config already prints an error if something goes wrong
|
|
_t=$("${CTDB_HELPER_BINDIR}/ctdb-config" get "$_section" "$_opt") ||
|
|
exit $?
|
|
echo "${_variable}=\"${_t}\""
|
|
}
|
|
|
|
######################################################
|
|
# wrapper around /proc/ settings to allow them to be hooked
|
|
# for testing
|
|
# 1st arg is relative path under /proc/, 2nd arg is value to set
|
|
set_proc()
|
|
{
|
|
echo "$2" >"/proc/$1"
|
|
}
|
|
|
|
set_proc_maybe()
|
|
{
|
|
if [ -w "/proc/$1" ]; then
|
|
set_proc "$1" "$2"
|
|
fi
|
|
}
|
|
|
|
######################################################
|
|
# wrapper around getting file contents from /proc/ to allow
|
|
# this to be hooked for testing
|
|
# 1st arg is relative path under /proc/
|
|
get_proc()
|
|
{
|
|
cat "/proc/$1"
|
|
}
|
|
|
|
######################################################
|
|
# Print up to $_max kernel stack traces for processes named $_program
|
|
program_stack_traces()
|
|
{
|
|
_prog="$1"
|
|
_max="${2:-1}"
|
|
|
|
_count=1
|
|
for _pid in $(pidof "$_prog"); do
|
|
[ "$_count" -le "$_max" ] || break
|
|
|
|
# Do this first to avoid racing with process exit
|
|
_stack=$(get_proc "${_pid}/stack" 2>/dev/null)
|
|
if [ -n "$_stack" ]; then
|
|
echo "Stack trace for ${_prog}[${_pid}]:"
|
|
echo "$_stack"
|
|
_count=$((_count + 1))
|
|
fi
|
|
done
|
|
}
|
|
|
|
######################################################
|
|
# Ensure $service_name is set
|
|
assert_service_name()
|
|
{
|
|
# service_name is set by the event script
|
|
# shellcheck disable=SC2154
|
|
[ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
|
|
}
|
|
|
|
######################################################
|
|
# check a set of directories is available
|
|
# return 1 on a missing directory
|
|
# directories are read from stdin
|
|
######################################################
|
|
ctdb_check_directories_probe()
|
|
{
|
|
while IFS="" read -r d; do
|
|
case "$d" in
|
|
*%*)
|
|
continue
|
|
;;
|
|
*)
|
|
[ -d "${d}/." ] || return 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
######################################################
|
|
# check a set of directories is available
|
|
# directories are read from stdin
|
|
######################################################
|
|
ctdb_check_directories()
|
|
{
|
|
ctdb_check_directories_probe || {
|
|
echo "ERROR: $service_name directory \"$d\" not available"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
######################################################
|
|
# check a set of tcp ports
|
|
# usage: ctdb_check_tcp_ports <ports...>
|
|
######################################################
|
|
|
|
# Check whether something is listening on all of the given TCP ports
|
|
# using the "ctdb checktcpport" command.
|
|
ctdb_check_tcp_ports()
|
|
{
|
|
if [ -z "$1" ]; then
|
|
echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
|
|
exit 1
|
|
fi
|
|
|
|
for _p; do # process each function argument (port)
|
|
_cmd="$CTDB checktcpport $_p"
|
|
_out=$($_cmd 2>&1)
|
|
_ret=$?
|
|
case "$_ret" in
|
|
0)
|
|
echo "$service_name not listening on TCP port $_p"
|
|
return 1
|
|
;;
|
|
98)
|
|
# Couldn't bind, something already listening, next port
|
|
continue
|
|
;;
|
|
*)
|
|
echo "unexpected error (${_ret}) running \"${_cmd}\""
|
|
if [ -n "$_out" ]; then
|
|
echo "$_out"
|
|
fi
|
|
return $_ret
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# All ports listening
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
# check a unix socket
|
|
# usage: ctdb_check_unix_socket SOCKPATH
|
|
######################################################
|
|
ctdb_check_unix_socket()
|
|
{
|
|
_sockpath="$1"
|
|
|
|
if [ -z "$_sockpath" ]; then
|
|
echo "ERROR: ctdb_check_unix_socket() requires socket path"
|
|
return 1
|
|
fi
|
|
|
|
_out=$(ss -l -xH "src ${_sockpath}")
|
|
if [ -z "$_out" ]; then
|
|
echo "ERROR: ${service_name} not listening on ${_sockpath}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
################################################
|
|
# kill off any TCP connections with the given IP
|
|
################################################
|
|
|
|
kill_tcp_summarise()
|
|
{
|
|
_mode="$1"
|
|
_count="$2"
|
|
_method="$3"
|
|
|
|
_connections=$(get_tcp_connections_for_ip "$_ip")
|
|
if [ -z "$_connections" ]; then
|
|
_remaining=0
|
|
else
|
|
_remaining=$(echo "$_connections" | wc -l)
|
|
fi
|
|
|
|
case "$_mode" in
|
|
total)
|
|
_total="$_count"
|
|
_killed=$((_total - _remaining))
|
|
;;
|
|
killed)
|
|
_killed="$_count"
|
|
_total=$((_killed + _remaining))
|
|
;;
|
|
esac
|
|
|
|
_t="${_killed}/${_total}"
|
|
_m=""
|
|
if [ -n "$_method" ]; then
|
|
_m=", using ${_method}"
|
|
fi
|
|
echo "Killed ${_t} TCP connections to released IP ${_ip}${_m}"
|
|
if [ -n "$_connections" ]; then
|
|
echo "Remaining connections:"
|
|
echo "$_connections" | sed -e 's|^| |'
|
|
fi
|
|
}
|
|
|
|
kill_tcp_connections()
|
|
{
|
|
_iface="$1"
|
|
_ip="$2"
|
|
|
|
_oneway=false
|
|
if [ "$3" = "oneway" ]; then
|
|
_oneway=true
|
|
fi
|
|
|
|
case "$CTDB_KILLTCP_USE_SS_KILL" in
|
|
yes | try)
|
|
_killcount=$(ss -K -tnH state established src "$_ip" | wc -l)
|
|
kill_tcp_summarise "killed" "$_killcount" "ss -K"
|
|
if [ "$CTDB_KILLTCP_USE_SS_KILL" = "yes" ]; then
|
|
return
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
get_tcp_connections_for_ip "$_ip" | {
|
|
_killcount=0
|
|
_connections=""
|
|
_nl="
|
|
"
|
|
while read -r _dst _src; do
|
|
_destport="${_dst##*:}"
|
|
__oneway=$_oneway
|
|
case $_destport in
|
|
# we only do one-way killtcp for CIFS
|
|
139 | 445) __oneway=true ;;
|
|
esac
|
|
|
|
_connections="${_connections}${_nl}${_src} ${_dst}"
|
|
if ! $__oneway; then
|
|
_connections="${_connections}${_nl}${_dst} ${_src}"
|
|
fi
|
|
|
|
_killcount=$((_killcount + 1))
|
|
done
|
|
|
|
if [ $_killcount -eq 0 ]; then
|
|
return
|
|
fi
|
|
|
|
if [ -n "$CTDB_KILLTCP_DEBUGLEVEL" ]; then
|
|
_debuglevel="$CTDB_KILLTCP_DEBUGLEVEL"
|
|
else
|
|
_debuglevel="$CTDB_DEBUGLEVEL"
|
|
fi
|
|
echo "$_connections" |
|
|
CTDB_DEBUGLEVEL="$_debuglevel" \
|
|
"${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
|
|
echo "Failed to kill TCP connections"
|
|
return
|
|
}
|
|
|
|
_method=""
|
|
if [ "$CTDB_KILLTCP_USE_SS_KILL" = "try" ]; then
|
|
_method="ctdb_killtcp"
|
|
fi
|
|
kill_tcp_summarise "total" "$_killcount" "$_method"
|
|
}
|
|
}
|
|
|
|
##################################################################
|
|
# kill off the local end for any TCP connections with the given IP
|
|
##################################################################
|
|
kill_tcp_connections_local_only()
|
|
{
|
|
kill_tcp_connections "$@" "oneway"
|
|
}
|
|
|
|
##################################################################
|
|
# tickle any TCP connections with the given IP
|
|
##################################################################
|
|
tickle_tcp_connections()
|
|
{
|
|
_ip="$1"
|
|
|
|
# Get connections, both directions
|
|
_conns=$(get_tcp_connections_for_ip "$_ip" |
|
|
awk '{ print $1, $2 ; print $2, $1 }')
|
|
|
|
echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
|
|
echo "$_conns" | ctdb tickle
|
|
}
|
|
|
|
get_tcp_connections_for_ip()
|
|
{
|
|
_ip="$1"
|
|
|
|
ss -tnH state established "src [$_ip]" | awk '{print $3, $4}'
|
|
}
|
|
|
|
########################################################
|
|
|
|
add_ip_to_iface()
|
|
{
|
|
_iface=$1
|
|
_ip=$2
|
|
_maskbits=$3
|
|
|
|
# Ensure interface is up
|
|
ip link set "$_iface" up ||
|
|
die "Failed to bringup interface $_iface"
|
|
|
|
# Only need to define broadcast for IPv4
|
|
case "$_ip" in
|
|
*:*) _bcast="" ;;
|
|
*) _bcast="brd +" ;;
|
|
esac
|
|
|
|
# Intentionally unquoted multi-word value here
|
|
# shellcheck disable=SC2086
|
|
ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
|
|
echo "Failed to add $_ip/$_maskbits on dev $_iface"
|
|
return 1
|
|
}
|
|
|
|
# Wait 5 seconds for IPv6 addresses to stop being tentative...
|
|
if [ -z "$_bcast" ]; then
|
|
for _x in $(seq 1 10); do
|
|
ip addr show to "${_ip}/128" | grep -q "tentative" || break
|
|
sleep 0.5
|
|
done
|
|
|
|
# If the address was a duplicate then it won't be on the
|
|
# interface so flag an error.
|
|
_t=$(ip addr show to "${_ip}/128")
|
|
case "$_t" in
|
|
"")
|
|
echo "Failed to add $_ip/$_maskbits on dev $_iface"
|
|
return 1
|
|
;;
|
|
*tentative* | *dadfailed*)
|
|
echo "Failed to add $_ip/$_maskbits on dev $_iface"
|
|
ip addr del "$_ip/$_maskbits" dev "$_iface"
|
|
return 1
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
delete_ip_from_iface()
|
|
{
|
|
_iface=$1
|
|
_ip=$2
|
|
_maskbits=$3
|
|
|
|
# This could be set globally for all interfaces but it is probably
|
|
# better to avoid surprises, so limit it the interfaces where CTDB
|
|
# has public IP addresses. There isn't anywhere else convenient
|
|
# to do this so just set it each time. This is much cheaper than
|
|
# remembering and re-adding secondaries.
|
|
set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
|
|
|
|
ip addr del "$_ip/$_maskbits" dev "$_iface" || {
|
|
echo "Failed to del $_ip on dev $_iface"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
# If the given IP is hosted then print 2 items: maskbits and iface
|
|
ip_maskbits_iface()
|
|
{
|
|
_addr="$1"
|
|
|
|
case "$_addr" in
|
|
*:*) _bits=128 ;;
|
|
*) _bits=32 ;;
|
|
esac
|
|
ip addr show to "${_addr}/${_bits}" 2>/dev/null |
|
|
awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
|
|
sub("@.*", "", iface) }
|
|
$1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
|
|
print mask, iface }'
|
|
}
|
|
|
|
drop_ip()
|
|
{
|
|
_addr="${1%/*}" # Remove optional maskbits
|
|
|
|
# Intentional word splitting here
|
|
# shellcheck disable=SC2046
|
|
set -- $(ip_maskbits_iface "$_addr")
|
|
if [ -n "$1" ]; then
|
|
_maskbits="$1"
|
|
_iface="$2"
|
|
echo "Removing public address $_addr/$_maskbits from device $_iface"
|
|
delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
|
|
fi
|
|
}
|
|
|
|
have_public_addresses()
|
|
{
|
|
[ -f "${CTDB_BASE}/public_addresses" ]
|
|
}
|
|
|
|
# This sets $public_ifaces as a side-effect.
|
|
get_public_ifaces()
|
|
{
|
|
# Get all the interfaces listed in the public_addresses file
|
|
public_ifaces=$(sed -e '/^#.*/d' \
|
|
-e 's/^[^\t ]*[\t ]*//' \
|
|
-e 's/,/ /g' \
|
|
-e 's/[\t ]*$//' "${CTDB_BASE}/public_addresses")
|
|
|
|
# Get the interfaces for which CTDB has public IPs configured.
|
|
# That is, for all but the 1st line, get the 1st field.
|
|
ctdb_ifaces=$($CTDB -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@')
|
|
|
|
# Add $ctdb_ifaces and make $public_ifaces unique
|
|
# Use word splitting to squash whitespace
|
|
# shellcheck disable=SC2086
|
|
public_ifaces=$(echo $public_ifaces $ctdb_ifaces | tr ' ' '\n' | sort -u)
|
|
}
|
|
|
|
drop_all_public_ips()
|
|
{
|
|
# _x is intentionally ignored
|
|
# shellcheck disable=SC2034
|
|
while read -r _ip _x; do
|
|
case "$_ip" in
|
|
\#*) continue ;;
|
|
esac
|
|
drop_ip "$_ip"
|
|
done <"${CTDB_BASE}/public_addresses"
|
|
}
|
|
|
|
flush_route_cache()
|
|
{
|
|
set_proc_maybe sys/net/ipv4/route/flush 1
|
|
set_proc_maybe sys/net/ipv6/route/flush 1
|
|
}
|
|
|
|
########################################################
|
|
# Interface monitoring
|
|
|
|
# If the interface is a virtual one (e.g. VLAN) then get the
|
|
# underlying interface
|
|
interface_get_real()
|
|
{
|
|
_iface="$1"
|
|
|
|
# If $_iface is a VLAN (i.e. contains an '@') then strip every
|
|
# before the '@', otherwise print the whole interface
|
|
echo "${_iface##*@}"
|
|
}
|
|
|
|
# Check whether an interface is operational
|
|
interface_monitor()
|
|
{
|
|
_iface="$1"
|
|
|
|
_iface_info=$(ip -br link show "$_iface" 2>&1) || {
|
|
echo "ERROR: Monitored interface ${_iface} does not exist"
|
|
return 1
|
|
}
|
|
|
|
# If the interface is a virtual one (e.g. VLAN) then get the
|
|
# underlying interface.
|
|
_realiface=$(interface_get_real "${_iface_info%% *}")
|
|
|
|
if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null); then
|
|
# This is a bond: various monitoring strategies
|
|
echo "$_bi" | grep -q 'Currently Active Slave: None' && {
|
|
echo "ERROR: No active slaves for bond device ${_realiface}"
|
|
return 1
|
|
}
|
|
echo "$_bi" | grep -q '^MII Status: up' || {
|
|
echo "ERROR: public network interface ${_realiface} is down"
|
|
return 1
|
|
}
|
|
echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
|
|
# This works around a bug in the driver where the
|
|
# overall bond status can be up but none of the actual
|
|
# physical interfaces have a link.
|
|
echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
|
|
echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
return 0
|
|
else
|
|
# Not a bond
|
|
case "$_iface" in
|
|
lo*)
|
|
# loopback is always working
|
|
return 0
|
|
;;
|
|
ib*)
|
|
# we don't know how to test ib links
|
|
return 0
|
|
;;
|
|
*)
|
|
ethtool "$_iface" | grep -q 'Link detected: yes' || {
|
|
# On some systems, this is not successful when a
|
|
# cable is plugged but the interface has not been
|
|
# brought up previously. Bring the interface up
|
|
# and try again...
|
|
ip link set "$_iface" up
|
|
ethtool "$_iface" | grep -q 'Link detected: yes' || {
|
|
echo "ERROR: No link on the public network interface ${_iface}"
|
|
return 1
|
|
}
|
|
}
|
|
return 0
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
########################################################
|
|
# Simple counters
|
|
_ctdb_counter_common()
|
|
{
|
|
[ $# -le 1 ] || die "usage: _ctdb_counter_common [name]"
|
|
|
|
if [ $# -eq 1 ]; then
|
|
_counter_name="${1}.failcount"
|
|
else
|
|
_counter_name="failcount"
|
|
fi
|
|
|
|
if [ -z "$script_state_dir" ]; then
|
|
die "ctdb_counter_* functions need ctdb_setup_state_dir()"
|
|
fi
|
|
|
|
_counter_file="${script_state_dir}/${_counter_name}"
|
|
}
|
|
# Some code passes an argument
|
|
# shellcheck disable=SC2120
|
|
ctdb_counter_init()
|
|
{
|
|
_ctdb_counter_common "$1"
|
|
|
|
: >"$_counter_file"
|
|
}
|
|
ctdb_counter_incr()
|
|
{
|
|
_ctdb_counter_common "$1"
|
|
|
|
# unary counting using newlines!
|
|
echo >>"$_counter_file"
|
|
}
|
|
ctdb_counter_get()
|
|
{
|
|
_ctdb_counter_common "$1"
|
|
# unary counting!
|
|
_val=$(wc -c 2>/dev/null <"$_counter_file" || echo 0)
|
|
# Strip leading spaces from output of wc (on freebsd)
|
|
# shellcheck disable=SC2086
|
|
echo $_val
|
|
}
|
|
ctdb_counter_exists()
|
|
{
|
|
_ctdb_counter_common "$1"
|
|
[ -e "$_counter_file" ]
|
|
}
|
|
|
|
#
|
|
# Fail counter/threshold combination to control warnings and node unhealthy
|
|
#
|
|
|
|
_failcount_validate_threshold()
|
|
{
|
|
case "$1" in
|
|
"") return 1 ;; # A failure that doesn't need a warning
|
|
*)
|
|
if echo "$1" | grep -qx '[0-9]*'; then
|
|
return 0
|
|
fi
|
|
|
|
echo "WARNING: ${1} is an invalid threshold in \"${2}\" check"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
_failcount_common()
|
|
{
|
|
_thing="$1"
|
|
|
|
_counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g')
|
|
}
|
|
|
|
failcount_init()
|
|
{
|
|
_thing="$1"
|
|
|
|
_failcount_common "$_thing"
|
|
|
|
ctdb_counter_init "$_counter"
|
|
}
|
|
|
|
failcount_reset()
|
|
{
|
|
_thing="$1"
|
|
|
|
_failcount_common "$_thing"
|
|
|
|
_failcount=$(ctdb_counter_get "$_counter")
|
|
if [ "$_failcount" -eq 0 ]; then
|
|
return
|
|
fi
|
|
|
|
printf 'NOTICE: %s: no longer failing\n' "$_thing"
|
|
ctdb_counter_init "$_counter"
|
|
}
|
|
|
|
failcount_incr()
|
|
{
|
|
_thing="$1"
|
|
_thresholds="$2"
|
|
_output="$3"
|
|
|
|
_failcount_common "$_thing"
|
|
|
|
ctdb_counter_incr "$_counter"
|
|
_failcount=$(ctdb_counter_get "$_counter")
|
|
|
|
case "$_thresholds" in
|
|
*:*)
|
|
_warn_threshold="${_thresholds%:*}"
|
|
_unhealthy_threshold="${_thresholds#*:}"
|
|
;;
|
|
"")
|
|
_warn_threshold=1
|
|
_unhealthy_threshold=""
|
|
;;
|
|
*)
|
|
_warn_threshold="$_thresholds"
|
|
_unhealthy_threshold=""
|
|
;;
|
|
esac
|
|
|
|
if _failcount_validate_threshold "$_unhealthy_threshold" "$_thing"; then
|
|
if [ "$_failcount" -ge "$_unhealthy_threshold" ]; then
|
|
printf 'ERROR: %s: fail count %d >= threshold %d\n' \
|
|
"$_thing" \
|
|
"$_failcount" \
|
|
"$_unhealthy_threshold"
|
|
# Only print output when exceeding the
|
|
# unhealthy threshold
|
|
if [ "$_failcount" -eq "$_unhealthy_threshold" ] && \
|
|
[ -n "$_output" ]; then
|
|
echo "$_output"
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if _failcount_validate_threshold "$_warn_threshold" "$_thing"; then
|
|
if [ "$_failcount" -lt "$_warn_threshold" ]; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
printf 'WARNING: %s: fail count %d >= threshold %d\n' \
|
|
"$_thing" \
|
|
"$_failcount" \
|
|
"$_warn_threshold"
|
|
if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then
|
|
# Only print output when exceeding the warning threshold
|
|
echo "$_output"
|
|
fi
|
|
}
|
|
|
|
########################################################
|
|
|
|
# ctdb_setup_state_dir <type> <name>
|
|
# Sets/creates script_state_dir)
|
|
ctdb_setup_state_dir()
|
|
{
|
|
[ $# -eq 2 ] || die "usage: ctdb_setup_state_dir <type> <name>"
|
|
|
|
_type="$1"
|
|
_name="$2"
|
|
|
|
script_state_dir="${CTDB_SCRIPT_VARDIR}/${_type}/${_name}"
|
|
|
|
mkdir -p "$script_state_dir" ||
|
|
die "Error creating script state dir \"${script_state_dir}\""
|
|
}
|
|
|
|
##################################################################
|
|
# Reconfigure a service on demand
|
|
|
|
_ctdb_service_reconfigure_common()
|
|
{
|
|
if [ -z "$script_state_dir" ]; then
|
|
die "ctdb_service_*_reconfigure() needs ctdb_setup_state_dir()"
|
|
fi
|
|
|
|
_ctdb_service_reconfigure_flag="${script_state_dir}/need_reconfigure"
|
|
}
|
|
|
|
ctdb_service_needs_reconfigure()
|
|
{
|
|
_ctdb_service_reconfigure_common
|
|
[ -e "$_ctdb_service_reconfigure_flag" ]
|
|
}
|
|
|
|
ctdb_service_set_reconfigure()
|
|
{
|
|
_ctdb_service_reconfigure_common
|
|
: >"$_ctdb_service_reconfigure_flag"
|
|
}
|
|
|
|
ctdb_service_unset_reconfigure()
|
|
{
|
|
_ctdb_service_reconfigure_common
|
|
rm -f "$_ctdb_service_reconfigure_flag"
|
|
}
|
|
|
|
ctdb_service_reconfigure()
|
|
{
|
|
echo "Reconfiguring service \"${service_name}\"..."
|
|
ctdb_service_unset_reconfigure
|
|
service_reconfigure || return $?
|
|
# Intentionally have this use $service_name as default
|
|
# shellcheck disable=SC2119
|
|
ctdb_counter_init
|
|
}
|
|
|
|
# Default service_reconfigure() function does nothing.
|
|
service_reconfigure()
|
|
{
|
|
:
|
|
}
|
|
|
|
# Default service_start() and service_stop() functions.
|
|
|
|
# These may be overridden in an eventscript.
|
|
service_start()
|
|
{
|
|
service "$service_name" start
|
|
}
|
|
|
|
service_stop()
|
|
{
|
|
service "$service_name" stop
|
|
}
|
|
|
|
##################################################################
|
|
|
|
# This exists only for backward compatibility with 3rd party scripts
|
|
# that call it
|
|
ctdb_standard_event_handler()
|
|
{
|
|
:
|
|
}
|
|
|
|
iptables_wrapper()
|
|
{
|
|
_family="$1"
|
|
shift
|
|
if [ "$_family" = "inet6" ]; then
|
|
_iptables_cmd="ip6tables"
|
|
else
|
|
_iptables_cmd="iptables"
|
|
fi
|
|
|
|
# iptables doesn't like being re-entered, so flock-wrap it.
|
|
flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
|
|
}
|
|
|
|
# AIX (and perhaps others?) doesn't have mktemp
|
|
# type is commonly supported and more portable than which(1)
|
|
# shellcheck disable=SC2039
|
|
if ! type mktemp >/dev/null 2>&1; then
|
|
mktemp()
|
|
{
|
|
_dir=false
|
|
if [ "$1" = "-d" ]; then
|
|
_dir=true
|
|
shift
|
|
fi
|
|
_d="${TMPDIR:-/tmp}"
|
|
_hex10=$(dd if=/dev/urandom count=20 2>/dev/null |
|
|
cksum |
|
|
awk '{print $1}')
|
|
_t="${_d}/tmp.${_hex10}"
|
|
(
|
|
umask 077
|
|
if $_dir; then
|
|
mkdir "$_t"
|
|
else
|
|
: >"$_t"
|
|
fi
|
|
)
|
|
echo "$_t"
|
|
}
|
|
fi
|
|
|
|
######################################################################
|
|
# NFS callout handling
|
|
|
|
nfs_callout_init()
|
|
{
|
|
_state_dir="$1"
|
|
|
|
if [ -z "$CTDB_NFS_CALLOUT" ]; then
|
|
CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
|
|
fi
|
|
# Always export, for statd callout
|
|
export CTDB_NFS_CALLOUT
|
|
|
|
# If the callout wants to use this then it must create it
|
|
export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"
|
|
|
|
# Export, if set, for use by clustered NFS callouts
|
|
if [ -n "$CTDB_NFS_STATE_FS_TYPE" ]; then
|
|
export CTDB_NFS_STATE_FS_TYPE
|
|
fi
|
|
if [ -n "$CTDB_NFS_STATE_MNT" ]; then
|
|
export CTDB_NFS_STATE_MNT
|
|
fi
|
|
if [ -n "$CTDB_NFS_EXPORTS_FILE" ]; then
|
|
export CTDB_NFS_EXPORTS_FILE
|
|
fi
|
|
|
|
nfs_callout_cache="${_state_dir}/nfs_callout_cache"
|
|
nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
|
|
nfs_callout_cache_ops="${nfs_callout_cache}/ops"
|
|
}
|
|
|
|
nfs_callout_register()
|
|
{
|
|
mkdir -p "$nfs_callout_cache_ops"
|
|
rm -f "$nfs_callout_cache_ops"/*
|
|
|
|
echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
|
|
|
|
_t=$("$CTDB_NFS_CALLOUT" "register")
|
|
if [ -n "$_t" ]; then
|
|
echo "$_t" |
|
|
while IFS="" read -r _op; do
|
|
touch "${nfs_callout_cache_ops}/${_op}"
|
|
done
|
|
else
|
|
touch "${nfs_callout_cache_ops}/ALL"
|
|
fi
|
|
}
|
|
|
|
nfs_callout()
|
|
{
|
|
# Re-run registration if $CTDB_NFS_CALLOUT has changed
|
|
_prev=""
|
|
if [ -r "$nfs_callout_cache_callout" ]; then
|
|
read -r _prev <"$nfs_callout_cache_callout"
|
|
fi
|
|
if [ "$CTDB_NFS_CALLOUT" != "$_prev" ]; then
|
|
nfs_callout_register
|
|
fi
|
|
|
|
# Run the operation if it is registered...
|
|
if [ -e "${nfs_callout_cache_ops}/${1}" ] ||
|
|
[ -e "${nfs_callout_cache_ops}/ALL" ]; then
|
|
"$CTDB_NFS_CALLOUT" "$@"
|
|
fi
|
|
}
|
|
|
|
########################################################
|
|
# tickle handling
|
|
########################################################
|
|
|
|
update_tickles()
|
|
{
|
|
tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
|
|
mkdir -p "$tickledir"
|
|
|
|
# If not hosting any public IPs then can't have any connections...
|
|
if [ ! -s "$CTDB_MY_PUBLIC_IPS_CACHE" ]; then
|
|
return
|
|
fi
|
|
|
|
# IPs ss filter
|
|
_ip_filter=""
|
|
while read -r _ip; do
|
|
_ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
|
|
done <"$CTDB_MY_PUBLIC_IPS_CACHE"
|
|
|
|
# Record our current tickles in a temporary file
|
|
_my_tickles="${tickledir}/all.tickles.$$"
|
|
while read -r _i; do
|
|
$CTDB -X gettickles "$_i" |
|
|
awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
|
|
done <"$CTDB_MY_PUBLIC_IPS_CACHE" |
|
|
sort >"$_my_tickles"
|
|
|
|
# Record connections to our public IPs in a temporary file.
|
|
# This temporary file is in CTDB's private state directory and
|
|
# $$ is used to avoid a very rare race involving CTDB's script
|
|
# debugging. No security issue, nothing to see here...
|
|
_my_connections="${tickledir}/all.connections.$$"
|
|
# Parentheses are needed around the IP filter for precedence but
|
|
# the parentheses can't be empty!
|
|
ss -tnH state established "${_ip_filter:+( ${_ip_filter} )}" |
|
|
awk '{print $4, $3}' |
|
|
sort >"$_my_connections"
|
|
|
|
# Add tickles for connections that we haven't already got tickles for
|
|
comm -23 "$_my_connections" "$_my_tickles" |
|
|
$CTDB addtickle
|
|
|
|
# Remove tickles for connections that are no longer there
|
|
comm -13 "$_my_connections" "$_my_tickles" |
|
|
$CTDB deltickle
|
|
|
|
rm -f "$_my_connections" "$_my_tickles"
|
|
|
|
# Remove stale files from killed scripts
|
|
# Files can't have spaces in name, more portable than -print0/-0
|
|
# shellcheck disable=SC2038
|
|
(cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
|
|
}
|
|
|
|
########################################################
|
|
# load a site local config file
|
|
########################################################
|
|
|
|
[ -x "${CTDB_BASE}/rc.local" ] && {
|
|
. "${CTDB_BASE}/rc.local"
|
|
}
|
|
|
|
[ -d "${CTDB_BASE}/rc.local.d" ] && {
|
|
for i in "${CTDB_BASE}/rc.local.d"/*; do
|
|
[ -x "$i" ] && . "$i"
|
|
done
|
|
}
|
|
|
|
script_name="${0##*/}" # basename
|