diff --git a/ctdb/config/events.d/60.nfs b/ctdb/config/events.d/60.nfs index 997d676091b..191af667a7f 100755 --- a/ctdb/config/events.d/60.nfs +++ b/ctdb/config/events.d/60.nfs @@ -7,16 +7,18 @@ . $CTDB_BASE/functions service_name="nfs" -service_start () + +if [ -z "$CTDB_NFS_CALLOUT" ] ; then + CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout" +fi +# Always export, for statd callout +export CTDB_NFS_CALLOUT + +nfs_callout () { - startstop_nfs stop - startstop_nfs start - set_proc "sys/net/ipv4/tcp_tw_recycle" 1 -} -service_stop () -{ - startstop_nfs stop + eval "$CTDB_NFS_CALLOUT" "$@" } + service_reconfigure () { # Restart lock manager, notify clients @@ -25,29 +27,6 @@ service_reconfigure () fi >/dev/null 2>&1 } -nfs_check_thread_count () -{ - [ "$CTDB_MONITOR_NFS_THREAD_COUNT" = "yes" ] || return 0 - - # If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could - # guess the default from the initscript. However, let's just - # assume that those using the default don't care about the number - # of threads and that they have switched on this feature in error. - _configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}" - [ -n "$_configured_threads" ] || return 0 - - # nfsd should be running the configured number of threads. If - # there are a different number of threads then tell nfsd the - # correct number. - _running_threads=$(get_proc "fs/nfsd/threads") - # Intentionally not arithmetic comparison - avoids extra errors - # when get_proc() fails... - if [ "$_running_threads" != "$_configured_threads" ] ; then - echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}" - set_proc "fs/nfsd/threads" "$_configured_threads" - fi -} - loadconfig [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] || exit 0 @@ -60,41 +39,40 @@ is_ctdb_managed_service || exit 0 ctdb_service_check_reconfigure -case "$1" in - init) - # read statd from persistent database - ;; - startup) - ctdb_service_start +case "$1" in + startup) + nfs_callout "$@" ;; - shutdown) - ctdb_service_stop + shutdown) + nfs_callout "$@" ;; - takeip) + takeip) + nfs_callout "$@" ctdb_service_set_reconfigure ;; - releaseip) + releaseip) + nfs_callout "$@" ctdb_service_set_reconfigure ;; - monitor) - # Check that directories for shares actually exist. - [ "$CTDB_NFS_SKIP_SHARE_CHECK" = "yes" ] || { - exportfs -v | grep '^/' | - sed -r -e 's@[[:space:]]+[^[:space:]()]+\([^[:space:]()]+\)$@@' | - sort -u | - ctdb_check_directories - } || exit $? + monitor) + nfs_callout "monitor-pre" || exit $? + + # Check that directories for shares actually exist + if [ "$CTDB_NFS_SKIP_SHARE_CHECK" != "yes" ] ; then + nfs_callout "monitor-list-shares" | ctdb_check_directories || \ + exit $? + fi update_tickles 2049 nfs_update_lock_info nfs_check_services - nfs_check_thread_count + nfs_callout "monitor-post" || exit $? ;; *) diff --git a/ctdb/config/nfs-checks.d/20.nfs.check b/ctdb/config/nfs-checks.d/20.nfs.check index 7229f7d9fe0..dad1cdc1136 100644 --- a/ctdb/config/nfs-checks.d/20.nfs.check +++ b/ctdb/config/nfs-checks.d/20.nfs.check @@ -2,6 +2,6 @@ version="3" restart_every=10 unhealthy_after=2 -service_stop_cmd="startstop_nfs restart-stop" -service_start_cmd="startstop_nfs start" +service_stop_cmd="$CTDB_NFS_CALLOUT stop nfs" +service_start_cmd="$CTDB_NFS_CALLOUT start nfs" service_debug_cmd="program_stack_traces nfsd 5" diff --git a/ctdb/config/nfs-checks.d/30.nlockmgr.check b/ctdb/config/nfs-checks.d/30.nlockmgr.check index c2e723e1051..6660ca0d91c 100644 --- a/ctdb/config/nfs-checks.d/30.nlockmgr.check +++ b/ctdb/config/nfs-checks.d/30.nlockmgr.check @@ -2,5 +2,5 @@ version="4" restart_every=2 unhealthy_after=6 -service_stop_cmd="startstop_nfslock stop" -service_start_cmd="startstop_nfslock start" +service_stop_cmd="$CTDB_NFS_CALLOUT stop nlockmgr" +service_start_cmd="$CTDB_NFS_CALLOUT start nlockmgr" diff --git a/ctdb/config/nfs-linux-kernel-callout b/ctdb/config/nfs-linux-kernel-callout new file mode 100755 index 00000000000..59618d79dfb --- /dev/null +++ b/ctdb/config/nfs-linux-kernel-callout @@ -0,0 +1,205 @@ +#!/bin/sh + +# Exit on 1st error +set -e + +# Red Hat +nfs_service="nfs" +nfslock_service="nfslock" +nfs_config="/etc/sysconfig/nfs" + +# SUSE +#nfs_service="nfsserver" +#nfslock_service="" +#nfs_config="/etc/sysconfig/nfs" + +# Debian +#nfs_service="nfs-kernel-server" +#nfslock_service="" +#nfs_config="/etc/default/nfs-kernel-server" + +# Override for unit testing +if [ -z "$PROCFS_PATH" ] ; then + PROCFS_PATH="/proc" +fi + +################################################## + +usage () +{ + _c=$(basename $0) + cat <"${PROCFS_PATH}/fs/nfsd/threads" + basic_stop "nfs" >/dev/null 2>&1 || true + pkill -9 nfsd + ;; + nlockmgr) + basic_stop "nfslock" >/dev/null 2>&1 || true + ;; + *) + usage + esac +} + +service_start () +{ + case "$1" in + nfs) + basic_start "nfs" + ;; + nlockmgr) + basic_start "nfslock" + ;; + *) + usage + esac +} + +################################################## +# service init startup and final shutdown + +nfs_shutdown () +{ + basic_stop "nfs" +} + +nfs_startup () +{ + basic_stop "nfs" || true + basic_start "nfs" + _f="${PROCFS_PATH}/sys/net/ipv4/tcp_tw_recycle" + if [ "$_f" ] ; then + echo 1 >"$_f" + fi +} + +################################################## +# monitor-post support + +nfs_check_thread_count () +{ + # Load NFS configuration to get desired number of threads. + if [ -r "$nfs_config" ] ; then + . "$nfs_config" + fi + + # If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could + # guess the default from the initscript. However, let's just + # assume that those using the default don't care about the number + # of threads and that they have switched on this feature in error. + _configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}" + [ -n "$_configured_threads" ] || return 0 + + _threads_file="${PROCFS_PATH}/fs/nfsd/threads" + + # nfsd should be running the configured number of threads. If + # there are a different number of threads then tell nfsd the + # correct number. + read _running_threads <"$_threads_file" + # Intentionally not arithmetic comparison - avoids extra errors + # when above fails... + if [ "$_running_threads" != "$_configured_threads" ] ; then + echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}" + echo "$_configured_threads" >"$_threads_file" + fi +} + +################################################## +# list share directories + +nfs_monitor_list_shares () +{ + exportfs -v | + grep '^/' | + sed -e 's@[[:space:]][[:space:]]*[^[:space:]()][^[:space:]()]*([^[:space:]()][^[:space:]()]*)$@@' | + sort -u +} + +################################################## + +case "$1" in + shutdown) + nfs_shutdown + ;; + startup) + nfs_startup + ;; + stop) + service_stop "$2" + ;; + start) + service_start "$2" + ;; + monitor-list-shares) + nfs_monitor_list_shares + ;; + monitor-post) + nfs_check_thread_count + ;; + monitor-pre|releaseip|takeip) + # Not required/implemented + : + ;; + *) + usage +esac diff --git a/ctdb/config/statd-callout b/ctdb/config/statd-callout index 347af41a7cd..53a2e4b7911 100755 --- a/ctdb/config/statd-callout +++ b/ctdb/config/statd-callout @@ -116,15 +116,15 @@ case "$1" in # service is started. state_even=$(( $(date '+%s') / 2 * 2)) - # we must also let some time pass between stopping and restarting the - # lockmanager since othervise there is a window where the lockmanager - # will respond "strangely" immediately after restarting it, which - # causes clients to fail to reclaim the locks. - # + # We must also let some time pass between stopping and + # restarting the lock manager. Otherwise there is a window + # where the lock manager will respond "strangely" immediately + # after restarting it, which causes clients to fail to reclaim + # their locks. if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then - startstop_nfslock stop >/dev/null 2>&1 + "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1 sleep 2 - startstop_nfslock start >/dev/null 2>&1 + "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1 fi # we now need to send out additional statd notifications to ensure diff --git a/ctdb/doc/ctdbd.conf.5.xml b/ctdb/doc/ctdbd.conf.5.xml index bb44317a42b..a0335832306 100644 --- a/ctdb/doc/ctdbd.conf.5.xml +++ b/ctdb/doc/ctdbd.conf.5.xml @@ -1055,6 +1055,21 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000 + + CTDB_NFS_CALLOUT=COMMAND + + + COMMAND specifies the path to a callout to handle + interactions with the configured NFS system, including + startup, shutdown, monitoring. + + + Default is the included + nfs-linux-kernel-callout. + + + + CTDB_NFS_DUMP_STUCK_THREADS=NUM diff --git a/ctdb/packaging/RPM/ctdb.spec.in b/ctdb/packaging/RPM/ctdb.spec.in index 503670023b8..25d2be7e2b5 100644 --- a/ctdb/packaging/RPM/ctdb.spec.in +++ b/ctdb/packaging/RPM/ctdb.spec.in @@ -191,6 +191,7 @@ rm -rf $RPM_BUILD_ROOT %config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/40.mountd.check %config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/50.rquotad.check %{_sysconfdir}/ctdb/statd-callout +%{_sysconfdir}/ctdb/nfs-linux-kernel-callout %{_sbindir}/ctdbd %{_sbindir}/ctdbd_wrapper %{_bindir}/ctdb diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.102.sh b/ctdb/tests/eventscripts/60.nfs.monitor.102.sh index bb988aa9a2d..509dc2a705e 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.102.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.102.sh @@ -8,7 +8,7 @@ setup_nfs CTDB_MONITOR_NFS_THREAD_COUNT="yes" RPCNFSDCOUNT=8 -FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6 7 8" +nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6 7 8 ok_null diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.103.sh b/ctdb/tests/eventscripts/60.nfs.monitor.103.sh index 75d7291fb85..2bf44f0efa9 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.103.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.103.sh @@ -8,7 +8,7 @@ setup_nfs CTDB_MONITOR_NFS_THREAD_COUNT="yes" RPCNFSDCOUNT=8 -FAKE_NFSD_THREAD_PIDS="1 2 3 4 5" +nfs_setup_fake_threads "nfsd" 1 2 3 4 5 ok "Attempting to correct number of nfsd threads from 5 to 8" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.104.sh b/ctdb/tests/eventscripts/60.nfs.monitor.104.sh index a052be81319..64831ee3434 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.104.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.104.sh @@ -11,7 +11,7 @@ setup_nfs CTDB_MONITOR_NFS_THREAD_COUNT="yes" RPCNFSDCOUNT=4 -FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6" +nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6 ok "Attempting to correct number of nfsd threads from 6 to 4" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.113.sh b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh index 711d5a9a1f8..dae9db3f8ed 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.113.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh @@ -10,6 +10,6 @@ rpc_services_down "nfs" # Additionally, any hung threads should have stack traces dumped. CTDB_NFS_DUMP_STUCK_THREADS=5 -FAKE_NFSD_THREAD_PIDS="" +nfs_setup_fake_threads "nfsd" nfs_iterate_test 10 "nfs" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.114.sh b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh index 55fe66d041b..00f30c5d740 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.114.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh @@ -10,6 +10,6 @@ rpc_services_down "nfs" # Additionally, any hung threads should have stack traces dumped. CTDB_NFS_DUMP_STUCK_THREADS=5 -FAKE_NFSD_THREAD_PIDS="1001 1002 1003" +nfs_setup_fake_threads "nfsd" 1001 1002 1003 nfs_iterate_test 10 "nfs" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.143.sh b/ctdb/tests/eventscripts/60.nfs.monitor.143.sh index 4ed0dcf474a..34a4c80e49c 100755 --- a/ctdb/tests/eventscripts/60.nfs.monitor.143.sh +++ b/ctdb/tests/eventscripts/60.nfs.monitor.143.sh @@ -9,6 +9,6 @@ define_test "statd down, 2 iterations, stuck process" setup_nfs rpc_services_down "status" CTDB_NFS_DUMP_STUCK_THREADS=2 -FAKE_RPC_THREAD_PIDS="1001" +nfs_setup_fake_threads "rpc.status" 1001 nfs_iterate_test 2 "status" diff --git a/ctdb/tests/eventscripts/etc-ctdb/nfs-linux-kernel-callout b/ctdb/tests/eventscripts/etc-ctdb/nfs-linux-kernel-callout new file mode 120000 index 00000000000..29858c24aa9 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/nfs-linux-kernel-callout @@ -0,0 +1 @@ +../../../config/nfs-linux-kernel-callout \ No newline at end of file diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index b13399e97c3..a1ba67d9981 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -789,8 +789,8 @@ setup_nfs () export CTDB_NFS_SKIP_SHARE_CHECK="no" - export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS - export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS + export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT + export CTDB_NFS_DUMP_STUCK_THREADS # Reset the failcounts for nfs services. eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' @@ -805,6 +805,9 @@ setup_nfs () export CTDB_MANAGED_SERVICES="foo nfs bar" rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" + + nfs_setup_fake_threads "nfsd" + nfs_setup_fake_threads "rpc.foobar" # Just set the variable to empty else debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" @@ -876,6 +879,24 @@ nfs_load_config () done } +nfs_setup_fake_threads () +{ + _prog="$1" ; shift + + case "$_prog" in + nfsd) + export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR") + _threads="${PROCFS_PATH}/fs/nfsd/threads" + mkdir -p $(dirname "$_threads") + echo $# >"$_threads" + export FAKE_NFSD_THREAD_PIDS="$*" + ;; + *) + export FAKE_RPC_THREAD_PIDS="$*" + ;; + esac +} + program_stack_traces () { _prog="$1" @@ -896,10 +917,10 @@ EOF guess_output () { case "$1" in - startstop_nfslock\ start) + $CTDB_NFS_CALLOUT\ start\ nlockmgr) echo "&Starting nfslock: OK" ;; - startstop_nfs\ start) + $CTDB_NFS_CALLOUT\ start\ nfs) cat <