mirror of
https://github.com/samba-team/samba.git
synced 2025-03-01 04:58:35 +03:00
78b7043411
Change the monitor event in 40.vsftpd so it only fails if there are 2 successive failures connecting to port 21. This reduces the likelihood of unhealthy nodes due to vsftpd being restarted for reconfiguration due to node failover or system reconfiguration. New eventscript functions ctdb_counter_init, ctdb_counter_incr, ctdb_counter_limit. These are used to count arbitrary things in eventscripts, depending on the eventscript name and a tag that is passed, and determine if a specified limit has been hit. They're good for counting failures! These functions are used in 40.vsftpd and also in 01.reclock - the latter used to do the counting without these functions. Signed-off-by: Martin Schwenke <martin@meltin.net> (This used to be ctdb commit cfe63636a163730ae9ad3554b78519b3c07d8896)
543 lines
15 KiB
Plaintext
543 lines
15 KiB
Plaintext
# utility functions for ctdb event scripts
|
|
|
|
#######################################
|
|
# pull in a system config file, if any
|
|
loadconfig() {
|
|
name="$1"
|
|
if [ -f /etc/sysconfig/$name ]; then
|
|
. /etc/sysconfig/$name
|
|
elif [ -f /etc/default/$name ]; then
|
|
. /etc/default/$name
|
|
elif [ -f $CTDB_BASE/sysconfig/$name ]; then
|
|
. $CTDB_BASE/sysconfig/$name
|
|
fi
|
|
}
|
|
|
|
##############################################################
|
|
# determine on what type of system (init style) we are running
|
|
detect_init_style() {
|
|
# only do detection if not already set:
|
|
test "x$CTDB_INIT_STYLE" != "x" && return
|
|
|
|
if [ -x /sbin/startproc ]; then
|
|
CTDB_INIT_STYLE="suse"
|
|
elif [ -x /sbin/start-stop-daemon ]; then
|
|
CTDB_INIT_STYLE="debian"
|
|
else
|
|
CTDB_INIT_STYLE="redhat"
|
|
fi
|
|
}
|
|
|
|
######################################################
|
|
# simulate /sbin/service on platforms that don't have it
|
|
service() {
|
|
service_name="$1"
|
|
op="$2"
|
|
|
|
# do nothing, when no service was specified
|
|
test "x$service_name" = "x" && return
|
|
|
|
if [ -x /sbin/service ]; then
|
|
/sbin/service "$service_name" "$op"
|
|
elif [ -x /etc/init.d/$service_name ]; then
|
|
/etc/init.d/$service_name "$op"
|
|
elif [ -x /etc/rc.d/init.d/$service_name ]; then
|
|
/etc/rc.d/init.d/$service_name "$op"
|
|
fi
|
|
}
|
|
|
|
######################################################
|
|
# simulate /sbin/service (niced) on platforms that don't have it
|
|
nice_service() {
|
|
service_name="$1"
|
|
op="$2"
|
|
|
|
# do nothing, when no service was specified
|
|
test "x$service_name" = "x" && return
|
|
|
|
if [ -x /sbin/service ]; then
|
|
nice /sbin/service "$service_name" "$op"
|
|
elif [ -x /etc/init.d/$service_name ]; then
|
|
nice /etc/init.d/$service_name "$op"
|
|
elif [ -x /etc/rc.d/init.d/$service_name ]; then
|
|
nice /etc/rc.d/init.d/$service_name "$op"
|
|
fi
|
|
}
|
|
|
|
######################################################
|
|
# wait for a command to return a zero exit status
|
|
# usage: ctdb_wait_command SERVICE_NAME <command>
|
|
######################################################
|
|
ctdb_wait_command() {
|
|
service_name="$1"
|
|
wait_cmd="$2"
|
|
[ -z "$wait_cmd" ] && return;
|
|
all_ok=0
|
|
echo "Waiting for service $service_name to start"
|
|
while [ $all_ok -eq 0 ]; do
|
|
$wait_cmd > /dev/null 2>&1 && all_ok=1
|
|
ctdb status > /dev/null 2>&1 || {
|
|
echo "ctdb daemon has died. Exiting wait for $service_name"
|
|
exit 1
|
|
}
|
|
[ $all_ok -eq 1 ] || sleep 1
|
|
done
|
|
echo "Local service $service_name is up"
|
|
}
|
|
|
|
|
|
######################################################
|
|
# wait for a set of tcp ports
|
|
# usage: ctdb_wait_tcp_ports SERVICE_NAME <ports...>
|
|
######################################################
|
|
ctdb_wait_tcp_ports() {
|
|
service_name="$1"
|
|
shift
|
|
wait_ports="$*"
|
|
[ -z "$wait_ports" ] && return;
|
|
all_ok=0
|
|
echo "Waiting for tcp service $service_name to start"
|
|
while [ $all_ok -eq 0 ]; do
|
|
all_ok=1
|
|
for p in $wait_ports; do
|
|
if [ -x /usr/bin/netcat ]; then
|
|
/usr/bin/netcat -z 127.0.0.1 $p > /dev/null || all_ok=0
|
|
elif [ -x /usr/bin/nc ]; then
|
|
/usr/bin/nc -z 127.0.0.1 $p > /dev/null || all_ok=0
|
|
elif [ -x /usr/bin/netstat ]; then
|
|
(netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
|
|
elif [ -x /bin/netstat ]; then
|
|
(netstat -a -n | egrep "0.0.0.0:$p[[:space:]]*LISTEN" > /dev/null) || all_ok=0
|
|
else
|
|
echo "No tool to check tcp ports availabe. can not check in ctdb_wait_tcp_ports"
|
|
return
|
|
fi
|
|
done
|
|
[ $all_ok -eq 1 ] || sleep 1
|
|
ctdb status > /dev/null 2>&1 || {
|
|
echo "ctdb daemon has died. Exiting tcp wait $service_name"
|
|
exit 1
|
|
}
|
|
done
|
|
echo "Local tcp services for $service_name are up"
|
|
}
|
|
|
|
|
|
|
|
######################################################
|
|
# wait for a set of directories
|
|
# usage: ctdb_wait_directories SERVICE_NAME <directories...>
|
|
######################################################
|
|
ctdb_wait_directories() {
|
|
service_name="$1"
|
|
shift
|
|
wait_dirs="$*"
|
|
[ -z "$wait_dirs" ] && return;
|
|
all_ok=0
|
|
echo "Waiting for local directories for $service_name"
|
|
while [ $all_ok -eq 0 ]; do
|
|
all_ok=1
|
|
for d in $wait_dirs; do
|
|
[ -d $d ] || all_ok=0
|
|
done
|
|
[ $all_ok -eq 1 ] || sleep 1
|
|
ctdb status > /dev/null 2>&1 || {
|
|
echo "ctdb daemon has died. Exiting directory wait for $service_name"
|
|
exit 1
|
|
}
|
|
done
|
|
echo "Local directories for $service_name are available"
|
|
}
|
|
|
|
|
|
######################################################
|
|
# check that a rpc server is registered with portmap
|
|
# and responding to requests
|
|
# usage: ctdb_check_rpc SERVICE_NAME PROGNUM VERSION
|
|
######################################################
|
|
ctdb_check_rpc() {
|
|
service_name="$1"
|
|
prognum="$2"
|
|
version="$3"
|
|
rpcinfo -u localhost $prognum $version > /dev/null || {
|
|
echo "ERROR: $service_name not responding to rpc requests"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
######################################################
|
|
# check a set of directories is available
|
|
# return 0 on a missing directory
|
|
# usage: ctdb_check_directories_probe SERVICE_NAME <directories...>
|
|
######################################################
|
|
ctdb_check_directories_probe() {
|
|
service_name="$1"
|
|
shift
|
|
wait_dirs="$*"
|
|
[ -z "$wait_dirs" ] && return;
|
|
for d in $wait_dirs; do
|
|
( echo $d | grep -q '%' ) && continue
|
|
[ -d $d ] || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
######################################################
|
|
# check a set of directories is available
|
|
# usage: ctdb_check_directories SERVICE_NAME <directories...>
|
|
######################################################
|
|
ctdb_check_directories() {
|
|
service_name="$1"
|
|
shift
|
|
wait_dirs="$*"
|
|
ctdb_check_directories_probe "$service_name" $wait_dirs || {
|
|
echo "ERROR: $service_name directory $d not available"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
######################################################
|
|
# check a set of tcp ports
|
|
# usage: ctdb_check_tcp_ports SERVICE_NAME <ports...>
|
|
######################################################
|
|
ctdb_check_tcp_ports() {
|
|
service_name="$1"
|
|
shift
|
|
wait_ports="$*"
|
|
[ -z "$wait_ports" ] && return;
|
|
|
|
# check availability of netcat or netstat first
|
|
NETCAT=""
|
|
NETSTAT=""
|
|
if [ -x /usr/bin/netstat ]; then
|
|
NETSTAT=/usr/bin/netstat
|
|
elif [ -x /bin/netstat ]; then
|
|
NETSTAT=/bin/netstat
|
|
elif [ -x /usr/bin/netcat ]; then
|
|
NETCAT=/usr/bin/netcat
|
|
elif [ -x /bin/netcat ]; then
|
|
NETCAT=/bin/netcat
|
|
elif [ -x /usr/bin/nc ]; then
|
|
NETCAT=/usr/bin/nc
|
|
elif [ -x /bin/nc ]; then
|
|
NETCAT=/bin/nc
|
|
fi
|
|
|
|
for p in $wait_ports; do
|
|
all_ok=1
|
|
|
|
if [ "x${NETCAT}" != "x" ]; then
|
|
${NETCAT} -z 127.0.0.1 $p > /dev/null || all_ok=0
|
|
elif [ "x${NETSTAT}" != "x" ]; then
|
|
if ! ${NETSTAT} -a -n | egrep "0.0.0.0:$p .*LISTEN" > /dev/null ; then
|
|
if ! ${NETSTAT} -a -n | egrep ":::$p .*LISTEN" > /dev/null ; then
|
|
all_ok=0
|
|
fi
|
|
fi
|
|
else
|
|
echo "ERROR: neither netcat (or nc) nor netstat found!"
|
|
echo "ERROR: can't monitor ${service_name} tcp port ${p}"
|
|
all_ok=0
|
|
fi
|
|
|
|
[ $all_ok -eq 1 ] || {
|
|
echo "ERROR: $service_name tcp port $p is not responding"
|
|
exit 1
|
|
}
|
|
done
|
|
}
|
|
|
|
######################################################
|
|
# check a unix socket
|
|
# usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
|
|
######################################################
|
|
ctdb_check_unix_socket() {
|
|
service_name="$1"
|
|
socket_path="$2"
|
|
[ -z "$socket_path" ] && return;
|
|
|
|
# check availability of netstat first
|
|
NETSTAT=""
|
|
if [ -x $(type -p netstat) ]; then
|
|
NETSTAT=$(type -p netstat)
|
|
elif [ -x /usr/bin/netstat ]; then
|
|
NETSTAT=/usr/bin/netstat
|
|
elif [ -x /bin/netstat ]; then
|
|
NETSTAT=/bin/netstat
|
|
fi
|
|
|
|
all_ok=1
|
|
if [ "x$NETSTAT" != "x" ]; then
|
|
if $NETSTAT -l -a -n | grep -qE "^unix.*LISTEN.*${socket_path}$"; then
|
|
all_ok=1
|
|
else
|
|
all_ok=0
|
|
fi
|
|
else
|
|
[ -S ${socket_path} ] && all_ok=1 || all_ok=0
|
|
fi
|
|
|
|
[ $all_ok -eq 1 ] || {
|
|
echo "ERROR: $service_name socket $socket_path not found"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
######################################################
|
|
# check a command returns zero status
|
|
# usage: ctdb_check_command SERVICE_NAME <command>
|
|
######################################################
|
|
ctdb_check_command() {
|
|
service_name="$1"
|
|
wait_cmd="$2"
|
|
[ -z "$wait_cmd" ] && return;
|
|
$wait_cmd > /dev/null 2>&1 || {
|
|
echo "ERROR: $service_name - $wait_cmd returned error"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
################################################
|
|
# kill off any TCP connections with the given IP
|
|
################################################
|
|
kill_tcp_connections() {
|
|
_IP="$1"
|
|
_failed=0
|
|
|
|
_killcount=0
|
|
connfile="$CTDB_BASE/state/connections.$_IP"
|
|
netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
|
|
netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
|
|
|
|
while read dest src; do
|
|
srcip=`echo $src | sed -e "s/:[^:]*$//"`
|
|
srcport=`echo $src | sed -e "s/^.*://"`
|
|
destip=`echo $dest | sed -e "s/:[^:]*$//"`
|
|
destport=`echo $dest | sed -e "s/^.*://"`
|
|
echo "Killing TCP connection $srcip:$srcport $destip:$destport"
|
|
ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
|
|
case $destport in
|
|
# we only do one-way killtcp for NFS and CIFS
|
|
139|445|2049) : ;;
|
|
# for all others we do 2-way
|
|
*)
|
|
ctdb killtcp $destip:$destport $srcip:$srcport >/dev/null 2>&1 || _failed=1
|
|
;;
|
|
esac
|
|
_killcount=`expr $_killcount + 1`
|
|
done < $connfile
|
|
/bin/rm -f $connfile
|
|
|
|
[ $_failed = 0 ] || {
|
|
echo "Failed to send killtcp control"
|
|
return;
|
|
}
|
|
[ $_killcount -gt 0 ] || {
|
|
return;
|
|
}
|
|
_count=0
|
|
while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
|
|
sleep 1
|
|
_count=`expr $_count + 1`
|
|
[ $_count -gt 3 ] && {
|
|
echo "Timed out killing tcp connections for IP $_IP"
|
|
return;
|
|
}
|
|
done
|
|
echo "killed $_killcount TCP connections to released IP $_IP"
|
|
}
|
|
|
|
##################################################################
|
|
# kill off the local end for any TCP connections with the given IP
|
|
##################################################################
|
|
kill_tcp_connections_local_only() {
|
|
_IP="$1"
|
|
_failed=0
|
|
|
|
_killcount=0
|
|
connfile="$CTDB_BASE/state/connections.$_IP"
|
|
netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' > $connfile
|
|
netstat -tn |egrep "^tcp.*[[:space:]]+::ffff:$_IP:.*ESTABLISHED" | awk '{print $4" "$5}' >> $connfile
|
|
|
|
while read dest src; do
|
|
srcip=`echo $src | sed -e "s/:[^:]*$//"`
|
|
srcport=`echo $src | sed -e "s/^.*://"`
|
|
destip=`echo $dest | sed -e "s/:[^:]*$//"`
|
|
destport=`echo $dest | sed -e "s/^.*://"`
|
|
echo "Killing TCP connection $srcip:$srcport $destip:$destport"
|
|
ctdb killtcp $srcip:$srcport $destip:$destport >/dev/null 2>&1 || _failed=1
|
|
_killcount=`expr $_killcount + 1`
|
|
done < $connfile
|
|
/bin/rm -f $connfile
|
|
|
|
[ $_failed = 0 ] || {
|
|
echo "Failed to send killtcp control"
|
|
return;
|
|
}
|
|
[ $_killcount -gt 0 ] || {
|
|
return;
|
|
}
|
|
_count=0
|
|
while netstat -tn |egrep "^tcp.*[[:space:]]+$_IP:.*ESTABLISHED" > /dev/null; do
|
|
sleep 1
|
|
_count=`expr $_count + 1`
|
|
[ $_count -gt 3 ] && {
|
|
echo "Timed out killing tcp connections for IP $_IP"
|
|
return;
|
|
}
|
|
done
|
|
echo "killed $_killcount TCP connections to released IP $_IP"
|
|
}
|
|
|
|
########################################################
|
|
# start/stop the nfs service on different platforms
|
|
########################################################
|
|
startstop_nfs() {
|
|
PLATFORM="unknown"
|
|
[ -x /etc/init.d/nfsserver ] && {
|
|
PLATFORM="sles"
|
|
}
|
|
[ -x /etc/init.d/nfslock ] && {
|
|
PLATFORM="rhel"
|
|
}
|
|
|
|
case $PLATFORM in
|
|
sles)
|
|
case $1 in
|
|
start)
|
|
service nfsserver start
|
|
;;
|
|
stop)
|
|
service nfsserver stop > /dev/null 2>&1
|
|
;;
|
|
esac
|
|
;;
|
|
rhel)
|
|
case $1 in
|
|
start)
|
|
service nfslock start
|
|
service nfs start
|
|
;;
|
|
stop)
|
|
service nfs stop > /dev/null 2>&1
|
|
service nfslock stop > /dev/null 2>&1
|
|
;;
|
|
esac
|
|
;;
|
|
*)
|
|
echo "Unknown platform. NFS is not supported with ctdb"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
########################################################
|
|
# start/stop the nfs lockmanager service on different platforms
|
|
########################################################
|
|
startstop_nfslock() {
|
|
PLATFORM="unknown"
|
|
[ -x /etc/init.d/nfsserver ] && {
|
|
PLATFORM="sles"
|
|
}
|
|
[ -x /etc/init.d/nfslock ] && {
|
|
PLATFORM="rhel"
|
|
}
|
|
|
|
case $PLATFORM in
|
|
sles)
|
|
# for sles there is no service for lockmanager
|
|
# so we instead just shutdown/restart nfs
|
|
case $1 in
|
|
start)
|
|
service nfsserver start
|
|
;;
|
|
stop)
|
|
service nfsserver stop > /dev/null 2>&1
|
|
;;
|
|
esac
|
|
;;
|
|
rhel)
|
|
case $1 in
|
|
start)
|
|
service nfslock start
|
|
;;
|
|
stop)
|
|
service nfslock stop > /dev/null 2>&1
|
|
;;
|
|
esac
|
|
;;
|
|
*)
|
|
echo "Unknown platform. NFS locking is not supported with ctdb"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
########################################################
|
|
# remove an ip address from an interface
|
|
########################################################
|
|
remove_ip() {
|
|
# the ip tool will delete all secondary IPs if this is the primary.
|
|
# To work around this _very_ annoying behaviour we have to keep a
|
|
# record of the secondaries and re-add them afterwards. yuck
|
|
secondaries=""
|
|
if ip addr list dev $2 primary | grep -q "inet $1 " ; then
|
|
secondaries=`ip addr list dev $2 secondary | grep " inet " | awk '{print $2}'`
|
|
fi
|
|
ip addr del $1 dev $2 >/dev/null 2>/dev/null || failed=1
|
|
[ -z "$secondaries" ] || {
|
|
for i in $secondaries; do
|
|
if ip addr list dev $2 | grep -q "inet $i" ; then
|
|
echo "kept secondary $i on dev $2"
|
|
else
|
|
echo "re-adding secondary address $i to dev $2"
|
|
ip addr add $i dev $2 || failed=1
|
|
fi
|
|
done
|
|
}
|
|
}
|
|
|
|
########################################################
|
|
# some simple logic for counting events - per eventscript
|
|
# usage: ctdb_counter_init <tag>
|
|
# ctdb_counter_incr <tag>
|
|
# ctdb_counter_limit <tag> <limit>
|
|
# e.g. <tag> = "fail-count"
|
|
# ctdb_counter_limit succeeds when count >= <limit>
|
|
########################################################
|
|
_ctdb_counter_common () {
|
|
_tag="$1"
|
|
_eventscript="${0##*/}" # basename
|
|
|
|
_counter_file="$CTDB_BASE/state/${_eventscript}-${_tag}"
|
|
mkdir -p "${_counter_file%/*}" # dirname
|
|
}
|
|
ctdb_counter_init () {
|
|
_ctdb_counter_common "$1"
|
|
|
|
echo -n > "$_counter_file"
|
|
}
|
|
ctdb_counter_incr () {
|
|
_ctdb_counter_common "$1"
|
|
|
|
# unary counting!
|
|
echo -n 1 >> "$_counter_file"
|
|
}
|
|
ctdb_counter_limit () {
|
|
_ctdb_counter_common "$1"
|
|
_limit="$2"
|
|
|
|
# unary counting!
|
|
_size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
|
|
[ $_size -ge $_limit ]
|
|
}
|
|
########################################################
|
|
# load a site local config file
|
|
########################################################
|
|
|
|
[ -x $CTDB_BASE/rc.local ] && {
|
|
. $CTDB_BASE/rc.local
|
|
}
|
|
|
|
|