1
0
mirror of https://github.com/samba-team/samba.git synced 2024-12-22 13:34:15 +03:00
samba-mirror/ctdb/config/statd-callout

211 lines
7.3 KiB
Plaintext
Raw Normal View History

#!/bin/sh
# This must run as root as CTDB tool commands need to access CTDB socket
[ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
# this script needs to be installed so that statd points to it with the -H
# command line argument. The easiest way to do that is to put something like this in
# /etc/sysconfig/nfs:
# STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
[ -n "$CTDB_BASE" ] || \
export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
. $CTDB_BASE/functions
# Overwrite this so we get some logging
die ()
{
script_log "statd-callout" "$@"
exit 1
}
loadconfig ctdb
loadconfig nfs
[ -n "$NFS_HOSTNAME" ] || \
die "NFS_HOSTNAME is not configured. statd-callout failed"
# A handy newline
nl="
"
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
ctdb_setup_service_state_dir "statd-callout"
cd "$service_state_dir" || \
die "Failed to change directory to \"${service_state_dir}\""
case "$1" in
# Keep a single file to keep track of the last "add-client" or
# "del-client'. These get pushed to ctdb.tdb during "update",
# which will generally be run once each "monitor" cycle. In this
# way we avoid scalability problems with flood of persistent
# transactions after a "notify" when all the clients re-take their
# locks.
add-client)
# statd does not tell us to which IP the client connected so
# we must add it to all the IPs that we serve
cip="$2"
ctdb_get_pnn
date=$(date '+%s')
ctdb ip -X |
tail -n +2 |
while IFS="|" read x sip node x ; do
[ "$node" = "$pnn" ] || continue # not us
key="statd-state@${sip}@${cip}"
echo "\"${key}\" \"${date}\"" >"$key"
done
;;
del-client)
# statd does not tell us from which IP the client disconnected
# so we must add it to all the IPs that we serve
cip="$2"
ctdb_get_pnn
ctdb ip -X |
tail -n +2 |
while IFS="|" read x sip node x ; do
[ "$node" = "$pnn" ] || continue # not us
key="statd-state@${sip}@${cip}"
echo "\"${key}\" \"\"" >"$key"
done
;;
update)
files=$(echo statd-state@*)
if [ "$files" = "statd-state@*" ] ; then
# No files!
exit 0
fi
# Filter out lines for any IP addresses that are not currently
# hosted public IP addresses.
ctdb_get_pnn
ctdb_ips=$(ctdb ip | tail -n +2)
sed_expr=$(echo "$ctdb_ips" |
awk -v pnn=$pnn 'pnn == $2 { \
ip = $1; gsub(/\./, "\\.", ip); \
printf "/statd-state@%s@/p\n", ip }')
if cat $files | sed -n "$sed_expr" | ctdb ptrans "ctdb.tdb" ; then
rm $files
fi
;;
notify)
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# we must restart the lockmanager (on all nodes) so that we get
# a clusterwide grace period (so other clients don't take out
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# conflicting locks through other nodes before all locks have been
# reclaimed)
# we need these settings to make sure that no tcp connections survive
# across a very fast failover/failback
#echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
#echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
#echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# Delete the notification list for statd, we don't want it to
# ping any clients
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
rm -f /var/lib/nfs/statd/sm/*
rm -f /var/lib/nfs/statd/sm.bak/*
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# we must keep a monotonically increasing state variable for the entire
# cluster so state always increases when ip addresses fail from one
# node to another
# We use epoch and hope the nodes are close enough in clock.
# Even numbers mean service is shut down, odd numbers mean
# service is started.
state_even=$(( $(date '+%s') / 2 * 2))
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# We must also let some time pass between stopping and
# restarting the lock manager. Otherwise there is a window
# where the lock manager will respond "strangely" immediately
# after restarting it, which causes clients to fail to reclaim
# their locks.
"$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
sleep 2
"$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
60.nfs: we must always restart the lockmanager when the cluster has been reconfigured and ip addresses has changed. This is to make sure we get a clusterwide grace period for nfs locking. if we dont do this and only restart locking on the nodes that were direclty affected, a different client can take out a conflicting lock from a different node before affected clients has had a chance to reclaim all the locks lost during reconfigure. grace period on rhel5 kernel has bene increased to 90 seconds! statd-callout: we must restart lockmanager to ensure a clusterwide grace period for nfs. this makes locking "more correct" for nfs clients and prevents other clients/nodes from taking out a conflicting lock while a different client/node tries to reclaim lost locks. This makes it "almost consistent" for NFS clients but there is still the possibility that a cifs client can take out a conflicting lock before an nfs client has had a chance to reclaim an existing lock. This can not be solved with anything less than making the kernel nfs lock manager "samba aware" and making samba aware of the internal state of the kernel lock manager so that they can cooperate. we can not just stop/start the lockmanager back to back in rhel5 since if they are stopped/started too close to eachother then when the new lockmanager upon starting up sends out statd notifications two things can happen: 1, new lockmanager sends out notification BEFORE it has registered with portmapper leading to lockmanager starts lockmanager sends notification to the client client tries to recover the lock and tries to portmap the lockmanager port on the server. server is not (yet) registered with portmapper and server responds "no such program" to hte clients request to discover where lockmanager is. client then just completely gives up reclaiming the lock and doesnt even reattempt the portmapper call after some timeout. ==> lock reclaim failed. 2, if they are started back to back, and a client tries to reclaim the lock the lockmanager sometimes sends two responses back to back to the client. one with status NLM_GRANTED (==you got the lock reclaimed) and one with status NLM_DENIED (==you could not get the lock reclaimed) This confuses the client and leads to the server thinking that the client does have the lock and the client thinking it has not got the lock and orphaned locks result. We also send out additional notification messages of different formats to allow more legacy clients to interoperate with locking. (This used to be ctdb commit 13208c1aab2942e28dff87e38e6794bf0c026033)
2007-09-07 02:52:56 +04:00
# we now need to send out additional statd notifications to ensure
# that clients understand that the lockmanager has restarted.
# we have three cases:
# 1, clients that ignore the ip address the stat notification came from
# and ONLY care about the 'name' in the notify packet.
# these clients ONLY work with lock failover IFF that name
# can be resolved into an ipaddress that matches the one used
# to mount the share. (==linux clients)
# This is handled when starting lockmanager above, but those
# packets are sent from the "wrong" ip address, something linux
# clients are ok with, buth other clients will barf at.
# 2, Some clients only accept statd packets IFF they come from the
# 'correct' ip address.
# 2a,Send out the notification using the 'correct' ip address and also
# specify the 'correct' hostname in the statd packet.
# Some clients require both the correct source address and also the
# correct name. (these clients also ONLY work if the ip addresses
# used to map the share can be resolved into the name returned in
# the notify packet.)
# 2b,Other clients require that the source ip address of the notify
# packet matches the ip address used to take out the lock.
# I.e. that the correct source address is used.
# These clients also require that the statd notify packet contains
# the name as the ip address used when the lock was taken out.
#
# Both 2a and 2b are commonly used in lockmanagers since they maximize
# probability that the client will accept the statd notify packet and
# not just ignore it.
# For all IPs we serve, collect info and push to the config database
ctdb_get_pnn
# Construct a sed expression to take catdb output and produce pairs of:
# server-IP client-IP
# but only for the server-IPs that are hosted on this node.
ctdb_all_ips=$(ctdb ip all | tail -n +2)
sed_expr=$(echo "$ctdb_all_ips" |
awk -v pnn=$pnn 'pnn == $2 { \
ip = $1; gsub(/\./, "\\.", ip); \
printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
[ -n "$statd_state" ] || exit 0
smnotify="${CTDB_HELPER_BINDIR}/smnotify"
prev=""
echo "$statd_state" | {
# This all needs to be in the same command group at the
# end of the pipe so it doesn't get lost when the loop
# completes.
items=""
while read sip cip ; do
# Collect item to delete from the DB
key="statd-state@${sip}@${cip}"
item="\"${key}\" \"\""
items="${items}${items:+${nl}}${item}"
# NOTE: Consider optimising smnotify to read all the
# data from stdin and then run it in the background.
# Reset stateval for each serverip
[ "$sip" = "$prev" ] || stateval="$state_even"
# Send notifies for server shutdown
"$smnotify" --client=$cip --ip=$sip --server=$sip --stateval=$stateval
"$smnotify" --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
# Send notifies for server startup
stateval=$(($stateval + 1))
"$smnotify" --client=$cip --ip=$sip --server=$sip --stateval=$stateval
"$smnotify" --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
done
echo "$items" | ctdb ptrans "ctdb.tdb"
}
# Remove any stale touch files (i.e. for IPs not currently
# hosted on this node and created since the last "update").
# There's nothing else we can do with them at this stage.
echo "$ctdb_all_ips" |
awk -v pnn=$pnn 'pnn != $2 { print $1 }' |
while read sip ; do
rm -f "statd-state@${sip}@"*
done
;;
esac