sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2024-12-22 13:34:15 +03:00
Code

ctdb-scripts: Parameterise 60.nfs with $CTDB_NFS_CALLOUT

Browse Source 
The goal is to have a single NFS eventscript.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
This commit is contained in:
Martin Schwenke 2015-06-24 21:36:14 +10:00 committed by Amitay Isaacs
parent 49c4d1900c
commit 9c87d1dd29
16 changed files with 294 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
ctdb/config/events.d/60.nfs
View File

@ -7,16 +7,18 @@
. $CTDB_BASE/functions
service_name="nfs"
service_start ()
if [ -z "$CTDB_NFS_CALLOUT" ] ; then
CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
fi
# Always export, for statd callout
export CTDB_NFS_CALLOUT
nfs_callout ()
{
startstop_nfs stop
startstop_nfs start
set_proc "sys/net/ipv4/tcp_tw_recycle" 1
}
service_stop ()
{
startstop_nfs stop
eval "$CTDB_NFS_CALLOUT" "$@"
}
service_reconfigure ()
{
# Restart lock manager, notify clients
@ -25,29 +27,6 @@ service_reconfigure ()
fi >/dev/null 2>&1
}
nfs_check_thread_count ()
{
[ "$CTDB_MONITOR_NFS_THREAD_COUNT" = "yes" ] || return 0
# If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could
# guess the default from the initscript. However, let's just
# assume that those using the default don't care about the number
# of threads and that they have switched on this feature in error.
_configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}"
[ -n "$_configured_threads" ] || return 0
# nfsd should be running the configured number of threads. If
# there are a different number of threads then tell nfsd the
# correct number.
_running_threads=$(get_proc "fs/nfsd/threads")
# Intentionally not arithmetic comparison - avoids extra errors
# when get_proc() fails...
if [ "$_running_threads" != "$_configured_threads" ] ; then
echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}"
set_proc "fs/nfsd/threads" "$_configured_threads"
fi
}
loadconfig
[ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] || exit 0
@ -60,41 +39,40 @@ is_ctdb_managed_service || exit 0
ctdb_service_check_reconfigure
case "$1" in
init)
# read statd from persistent database
;;
startup)
ctdb_service_start
case "$1" in
startup)
nfs_callout "$@"
;;
shutdown)
ctdb_service_stop
shutdown)
nfs_callout "$@"
;;
takeip)
takeip)
nfs_callout "$@"
ctdb_service_set_reconfigure
;;
releaseip)
releaseip)
nfs_callout "$@"
ctdb_service_set_reconfigure
;;
monitor)
# Check that directories for shares actually exist.
[ "$CTDB_NFS_SKIP_SHARE_CHECK" = "yes" ] || {
exportfs -v | grep '^/' |
sed -r -e 's@[[:space:]]+[^[:space:]()]+\([^[:space:]()]+\)$@@' |
sort -u |
ctdb_check_directories
} || exit $?
monitor)
nfs_callout "monitor-pre" || exit $?
# Check that directories for shares actually exist
if [ "$CTDB_NFS_SKIP_SHARE_CHECK" != "yes" ] ; then
nfs_callout "monitor-list-shares" | ctdb_check_directories || \
exit $?
fi
update_tickles 2049
nfs_update_lock_info
nfs_check_services
nfs_check_thread_count
nfs_callout "monitor-post" || exit $?
;;
*)

4
ctdb/config/nfs-checks.d/20.nfs.check
View File

@ -2,6 +2,6 @@
version="3"
restart_every=10
unhealthy_after=2
service_stop_cmd="startstop_nfs restart-stop"
service_start_cmd="startstop_nfs start"
service_stop_cmd="$CTDB_NFS_CALLOUT stop nfs"
service_start_cmd="$CTDB_NFS_CALLOUT start nfs"
service_debug_cmd="program_stack_traces nfsd 5"

4
ctdb/config/nfs-checks.d/30.nlockmgr.check
View File

@ -2,5 +2,5 @@
version="4"
restart_every=2
unhealthy_after=6
service_stop_cmd="startstop_nfslock stop"
service_start_cmd="startstop_nfslock start"
service_stop_cmd="$CTDB_NFS_CALLOUT stop nlockmgr"
service_start_cmd="$CTDB_NFS_CALLOUT start nlockmgr"

205
ctdb/config/nfs-linux-kernel-callout Executable file
View File

@ -0,0 +1,205 @@
#!/bin/sh
# Exit on 1st error
set -e
# Red Hat
nfs_service="nfs"
nfslock_service="nfslock"
nfs_config="/etc/sysconfig/nfs"
# SUSE
#nfs_service="nfsserver"
#nfslock_service=""
#nfs_config="/etc/sysconfig/nfs"
# Debian
#nfs_service="nfs-kernel-server"
#nfslock_service=""
#nfs_config="/etc/default/nfs-kernel-server"
# Override for unit testing
if [ -z "$PROCFS_PATH" ] ; then
PROCFS_PATH="/proc"
fi
##################################################
usage ()
{
_c=$(basename $0)
cat <<EOF
usage: $_c { shutdown | startup }
$_c { stop | start } { nfs | nlockmgr }
$_c { monitor-list-shares | monitor-post }
EOF
exit 1
}
##################################################
# Basic service stop and start
basic_stop ()
{
case "$1" in
nfs)
service "$nfs_service" stop
if [ -n "$nfslock_service" ] ; then
service "$nfslock_service" stop
fi
;;
nfslock)
if [ -n "$nfslock_service" ] ; then
service "$nfslock_service" stop
else
service "$nfs_service" stop
fi
;;
*)
usage
esac
}
basic_start ()
{
case "$1" in
nfs)
if [ -n "$nfslock_service" ] ; then
service "$nfslock_service" start
fi
service "$nfs_service" start
;;
nfslock)
if [ -n "$nfslock_service" ] ; then
service "$nfslock_service" start
else
service "$nfs_service" start
fi
;;
*)
usage
esac
}
##################################################
# service "stop" and "start" options for restarting
service_stop ()
{
case "$1" in
nfs)
echo 0 >"${PROCFS_PATH}/fs/nfsd/threads"
basic_stop "nfs" >/dev/null 2>&1 || true
pkill -9 nfsd
;;
nlockmgr)
basic_stop "nfslock" >/dev/null 2>&1 || true
;;
*)
usage
esac
}
service_start ()
{
case "$1" in
nfs)
basic_start "nfs"
;;
nlockmgr)
basic_start "nfslock"
;;
*)
usage
esac
}
##################################################
# service init startup and final shutdown
nfs_shutdown ()
{
basic_stop "nfs"
}
nfs_startup ()
{
basic_stop "nfs" || true
basic_start "nfs"
_f="${PROCFS_PATH}/sys/net/ipv4/tcp_tw_recycle"
if [ "$_f" ] ; then
echo 1 >"$_f"
fi
}
##################################################
# monitor-post support
nfs_check_thread_count ()
{
# Load NFS configuration to get desired number of threads.
if [ -r "$nfs_config" ] ; then
. "$nfs_config"
fi
# If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could
# guess the default from the initscript. However, let's just
# assume that those using the default don't care about the number
# of threads and that they have switched on this feature in error.
_configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}"
[ -n "$_configured_threads" ] || return 0
_threads_file="${PROCFS_PATH}/fs/nfsd/threads"
# nfsd should be running the configured number of threads. If
# there are a different number of threads then tell nfsd the
# correct number.
read _running_threads <"$_threads_file"
# Intentionally not arithmetic comparison - avoids extra errors
# when above fails...
if [ "$_running_threads" != "$_configured_threads" ] ; then
echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}"
echo "$_configured_threads" >"$_threads_file"
fi
}
##################################################
# list share directories
nfs_monitor_list_shares ()
{
exportfs -v |
grep '^/' |
sed -e 's@[[:space:]][[:space:]]*[^[:space:]()][^[:space:]()]*([^[:space:]()][^[:space:]()]*)$@@' |
sort -u
}
##################################################
case "$1" in
shutdown)
nfs_shutdown
;;
startup)
nfs_startup
;;
stop)
service_stop "$2"
;;
start)
service_start "$2"
;;
monitor-list-shares)
nfs_monitor_list_shares
;;
monitor-post)
nfs_check_thread_count
;;
monitor-pre|releaseip|takeip)
# Not required/implemented
:
;;
*)
usage
esac

14
ctdb/config/statd-callout
View File

@ -116,15 +116,15 @@ case "$1" in
# service is started.
state_even=$(( $(date '+%s') / 2 * 2))
# we must also let some time pass between stopping and restarting the
# lockmanager since othervise there is a window where the lockmanager
# will respond "strangely" immediately after restarting it, which
# causes clients to fail to reclaim the locks.
#
# We must also let some time pass between stopping and
# restarting the lock manager. Otherwise there is a window
# where the lock manager will respond "strangely" immediately
# after restarting it, which causes clients to fail to reclaim
# their locks.
if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then
startstop_nfslock stop >/dev/null 2>&1
"$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
sleep 2
startstop_nfslock start >/dev/null 2>&1
"$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
fi
# we now need to send out additional statd notifications to ensure

15
ctdb/doc/ctdbd.conf.5.xml
View File

@ -1055,6 +1055,21 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
</listitem>
</varlistentry>
<varlistentry>
<term>CTDB_NFS_CALLOUT=<parameter>COMMAND</parameter></term>
<listitem>
<para>
COMMAND specifies the path to a callout to handle
interactions with the configured NFS system, including
startup, shutdown, monitoring.
</para>
<para>
Default is the included
<command>nfs-linux-kernel-callout</command>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>CTDB_NFS_DUMP_STUCK_THREADS=<parameter>NUM</parameter></term>
<listitem>

1
ctdb/packaging/RPM/ctdb.spec.in
View File

@ -191,6 +191,7 @@ rm -rf $RPM_BUILD_ROOT
%config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/40.mountd.check
%config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/50.rquotad.check
%{_sysconfdir}/ctdb/statd-callout
%{_sysconfdir}/ctdb/nfs-linux-kernel-callout
%{_sbindir}/ctdbd
%{_sbindir}/ctdbd_wrapper
%{_bindir}/ctdb

2
ctdb/tests/eventscripts/60.nfs.monitor.102.sh
View File

@ -8,7 +8,7 @@ setup_nfs
CTDB_MONITOR_NFS_THREAD_COUNT="yes"
RPCNFSDCOUNT=8
FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6 7 8"
nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6 7 8
ok_null

2
ctdb/tests/eventscripts/60.nfs.monitor.103.sh
View File

@ -8,7 +8,7 @@ setup_nfs
CTDB_MONITOR_NFS_THREAD_COUNT="yes"
RPCNFSDCOUNT=8
FAKE_NFSD_THREAD_PIDS="1 2 3 4 5"
nfs_setup_fake_threads "nfsd" 1 2 3 4 5
ok "Attempting to correct number of nfsd threads from 5 to 8"

2
ctdb/tests/eventscripts/60.nfs.monitor.104.sh
View File

@ -11,7 +11,7 @@ setup_nfs
CTDB_MONITOR_NFS_THREAD_COUNT="yes"
RPCNFSDCOUNT=4
FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6"
nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6
ok "Attempting to correct number of nfsd threads from 6 to 4"

2
ctdb/tests/eventscripts/60.nfs.monitor.113.sh
View File

@ -10,6 +10,6 @@ rpc_services_down "nfs"
# Additionally, any hung threads should have stack traces dumped.
CTDB_NFS_DUMP_STUCK_THREADS=5
FAKE_NFSD_THREAD_PIDS=""
nfs_setup_fake_threads "nfsd"
nfs_iterate_test 10 "nfs"

2
ctdb/tests/eventscripts/60.nfs.monitor.114.sh
View File

@ -10,6 +10,6 @@ rpc_services_down "nfs"
# Additionally, any hung threads should have stack traces dumped.
CTDB_NFS_DUMP_STUCK_THREADS=5
FAKE_NFSD_THREAD_PIDS="1001 1002 1003"
nfs_setup_fake_threads "nfsd" 1001 1002 1003
nfs_iterate_test 10 "nfs"

2
ctdb/tests/eventscripts/60.nfs.monitor.143.sh
View File

@ -9,6 +9,6 @@ define_test "statd down, 2 iterations, stuck process"
setup_nfs
rpc_services_down "status"
CTDB_NFS_DUMP_STUCK_THREADS=2
FAKE_RPC_THREAD_PIDS="1001"
nfs_setup_fake_threads "rpc.status" 1001
nfs_iterate_test 2 "status"

1
ctdb/tests/eventscripts/etc-ctdb/nfs-linux-kernel-callout Symbolic link
View File

@ -0,0 +1 @@
../../../config/nfs-linux-kernel-callout

29
ctdb/tests/eventscripts/scripts/local.sh
View File

@ -789,8 +789,8 @@ setup_nfs ()
export CTDB_NFS_SKIP_SHARE_CHECK="no"
export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS
export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT
export CTDB_NFS_DUMP_STUCK_THREADS
# Reset the failcounts for nfs services.
eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
@ -805,6 +805,9 @@ setup_nfs ()
export CTDB_MANAGED_SERVICES="foo nfs bar"
rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
nfs_setup_fake_threads "nfsd"
nfs_setup_fake_threads "rpc.foobar" # Just set the variable to empty
else
debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
@ -876,6 +879,24 @@ nfs_load_config ()
done
}
nfs_setup_fake_threads ()
{
_prog="$1" ; shift
case "$_prog" in
nfsd)
export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
_threads="${PROCFS_PATH}/fs/nfsd/threads"
mkdir -p $(dirname "$_threads")
echo $# >"$_threads"
export FAKE_NFSD_THREAD_PIDS="$*"
;;
*)
export FAKE_RPC_THREAD_PIDS="$*"
;;
esac
}
program_stack_traces ()
{
_prog="$1"
@ -896,10 +917,10 @@ EOF
guess_output ()
{
case "$1" in
startstop_nfslock\ start)
$CTDB_NFS_CALLOUT\ start\ nlockmgr)
echo "&Starting nfslock: OK"
;;
startstop_nfs\ start)
$CTDB_NFS_CALLOUT\ start\ nfs)
cat <<EOF
&Starting nfslock: OK
&Starting nfs: OK

2
ctdb/wscript
View File

@ -486,6 +486,7 @@ def build(bld):
'debug-hung-script.sh',
'debug_locks.sh',
'gcore_trace.sh',
'nfs-linux-kernel-callout',
'notify.sh',
'statd-callout'
]
@ -628,6 +629,7 @@ def build(bld):
'events.d',
'functions',
'nfs-checks.d',
'nfs-linux-kernel-callout',
'statd-callout'
]