ctdb-scripts: Parameterise 60.nfs with $CTDB_NFS_CALLOUT

The goal is to have a single NFS eventscript.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

ctdb/config/events.d/60.nfs

@@ -7,16 +7,18 @@
 . $CTDB_BASE/functions
 
 service_name="nfs"
-service_start ()
+
+if [ -z "$CTDB_NFS_CALLOUT" ] ; then
+    CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
+fi
+# Always export, for statd callout
+export CTDB_NFS_CALLOUT
+
+nfs_callout ()
 {
-    startstop_nfs stop
+    eval "$CTDB_NFS_CALLOUT" "$@"
-    startstop_nfs start
-    set_proc "sys/net/ipv4/tcp_tw_recycle" 1
-}
-service_stop ()
-{
-    startstop_nfs stop
 }
+
 service_reconfigure ()
 {
     # Restart lock manager, notify clients
@@ -25,29 +27,6 @@ service_reconfigure ()
     fi >/dev/null 2>&1
 }
 
-nfs_check_thread_count ()
-{
-    [ "$CTDB_MONITOR_NFS_THREAD_COUNT" = "yes" ] || return 0
-
-    # If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could
-    # guess the default from the initscript.  However, let's just
-    # assume that those using the default don't care about the number
-    # of threads and that they have switched on this feature in error.
-    _configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}"
-    [ -n "$_configured_threads" ] || return 0
-
-    # nfsd should be running the configured number of threads.  If
-    # there are a different number of threads then tell nfsd the
-    # correct number.  
-    _running_threads=$(get_proc "fs/nfsd/threads")
-    # Intentionally not arithmetic comparison - avoids extra errors
-    # when get_proc() fails...
-    if [ "$_running_threads" != "$_configured_threads" ] ; then
-	echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}"
-	set_proc "fs/nfsd/threads" "$_configured_threads"
-    fi
-}
-
 loadconfig
 
 [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] || exit 0
@@ -61,40 +40,39 @@ is_ctdb_managed_service || exit 0
 ctdb_service_check_reconfigure
 
 case "$1" in
-     init)
+    startup)
-	# read statd from persistent database
+	nfs_callout "$@"
-	;;
-     startup)
-	ctdb_service_start
 	;;
 
-     shutdown)
+    shutdown)
-	ctdb_service_stop
+	 nfs_callout "$@"
 	;;
 
-     takeip)
+    takeip)
+	nfs_callout "$@"
 	ctdb_service_set_reconfigure
 	;;
 
-     releaseip)
+    releaseip)
+	nfs_callout "$@"
 	ctdb_service_set_reconfigure
 	;;
 
-      monitor)
+    monitor)
-	# Check that directories for shares actually exist.
+	nfs_callout "monitor-pre" || exit $?
-	[ "$CTDB_NFS_SKIP_SHARE_CHECK" = "yes" ] || {
+
-	    exportfs -v | grep '^/' | 
+	# Check that directories for shares actually exist
-	    sed -r -e 's@[[:space:]]+[^[:space:]()]+\([^[:space:]()]+\)$@@' | 
+	if [ "$CTDB_NFS_SKIP_SHARE_CHECK" != "yes" ] ; then
-	    sort -u | 
+	    nfs_callout "monitor-list-shares" | ctdb_check_directories || \
-	    ctdb_check_directories 
+		exit $?
-	} || exit $?
+	fi
 
 	update_tickles 2049
 	nfs_update_lock_info
 
 	nfs_check_services
 
-	nfs_check_thread_count
+	nfs_callout "monitor-post" || exit $?
        	;;
 
     *)

ctdb/config/nfs-checks.d/20.nfs.check

@@ -2,6 +2,6 @@
 version="3"
 restart_every=10
 unhealthy_after=2
-service_stop_cmd="startstop_nfs restart-stop"
+service_stop_cmd="$CTDB_NFS_CALLOUT stop nfs"
-service_start_cmd="startstop_nfs start"
+service_start_cmd="$CTDB_NFS_CALLOUT start nfs"
 service_debug_cmd="program_stack_traces nfsd 5"

ctdb/config/nfs-checks.d/30.nlockmgr.check

@@ -2,5 +2,5 @@
 version="4"
 restart_every=2
 unhealthy_after=6
-service_stop_cmd="startstop_nfslock stop"
+service_stop_cmd="$CTDB_NFS_CALLOUT stop nlockmgr"
-service_start_cmd="startstop_nfslock start"
+service_start_cmd="$CTDB_NFS_CALLOUT start nlockmgr"

ctdb/config/nfs-linux-kernel-callout

@@ -0,0 +1,205 @@
+#!/bin/sh
+
+# Exit on 1st error
+set -e
+
+# Red Hat
+nfs_service="nfs"
+nfslock_service="nfslock"
+nfs_config="/etc/sysconfig/nfs"
+
+# SUSE
+#nfs_service="nfsserver"
+#nfslock_service=""
+#nfs_config="/etc/sysconfig/nfs"
+
+# Debian
+#nfs_service="nfs-kernel-server"
+#nfslock_service=""
+#nfs_config="/etc/default/nfs-kernel-server"
+
+# Override for unit testing
+if [ -z "$PROCFS_PATH" ] ; then
+    PROCFS_PATH="/proc"
+fi
+
+##################################################
+
+usage ()
+{
+    _c=$(basename $0)
+    cat <<EOF
+usage: $_c { shutdown | startup }
+       $_c { stop | start } { nfs | nlockmgr }
+       $_c { monitor-list-shares | monitor-post }
+EOF
+    exit 1
+}
+
+
+##################################################
+# Basic service stop and start
+
+basic_stop ()
+{
+    case "$1" in
+	nfs)
+	    service "$nfs_service" stop
+	    if [ -n "$nfslock_service" ] ; then
+		service "$nfslock_service" stop
+	    fi
+	    ;;
+	nfslock)
+	    if [ -n "$nfslock_service" ] ; then
+		service "$nfslock_service" stop
+	    else
+		service "$nfs_service" stop
+	    fi
+	    ;;
+	*)
+	    usage
+    esac
+}
+
+basic_start ()
+{
+    case "$1" in
+	nfs)
+	    if [ -n "$nfslock_service" ] ; then
+		service "$nfslock_service" start
+	    fi
+	    service "$nfs_service" start
+	    ;;
+	nfslock)
+	    if [ -n "$nfslock_service" ] ; then
+		service "$nfslock_service" start
+	    else
+		service "$nfs_service" start
+	    fi
+	    ;;
+	*)
+	    usage
+    esac
+}
+
+##################################################
+# service "stop" and "start" options for restarting
+
+service_stop ()
+{
+    case "$1" in
+	nfs)
+	    echo 0 >"${PROCFS_PATH}/fs/nfsd/threads"
+	    basic_stop "nfs" >/dev/null 2>&1 || true
+	    pkill -9 nfsd
+	    ;;
+	nlockmgr)
+	    basic_stop "nfslock" >/dev/null 2>&1 || true
+	    ;;
+	*)
+	    usage
+    esac
+}
+
+service_start ()
+{
+    case "$1" in
+	nfs)
+	    basic_start "nfs"
+	    ;;
+	nlockmgr)
+	    basic_start "nfslock"
+	    ;;
+	*)
+	    usage
+    esac
+}
+
+##################################################
+# service init startup and final shutdown
+
+nfs_shutdown ()
+{
+    basic_stop "nfs"
+}
+
+nfs_startup ()
+{
+    basic_stop "nfs" || true
+    basic_start "nfs"
+    _f="${PROCFS_PATH}/sys/net/ipv4/tcp_tw_recycle"
+    if [ "$_f" ] ; then
+	echo 1 >"$_f"
+    fi
+}
+
+##################################################
+# monitor-post support
+
+nfs_check_thread_count ()
+{
+    # Load NFS configuration to get desired number of threads.
+    if [ -r "$nfs_config" ] ; then
+	. "$nfs_config"
+    fi
+
+    # If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could
+    # guess the default from the initscript.  However, let's just
+    # assume that those using the default don't care about the number
+    # of threads and that they have switched on this feature in error.
+    _configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}"
+    [ -n "$_configured_threads" ] || return 0
+
+    _threads_file="${PROCFS_PATH}/fs/nfsd/threads"
+
+    # nfsd should be running the configured number of threads.  If
+    # there are a different number of threads then tell nfsd the
+    # correct number.
+    read _running_threads <"$_threads_file"
+    # Intentionally not arithmetic comparison - avoids extra errors
+    # when above fails...
+    if [ "$_running_threads" != "$_configured_threads" ] ; then
+	echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}"
+	echo "$_configured_threads" >"$_threads_file"
+    fi
+}
+
+##################################################
+# list share directories
+
+nfs_monitor_list_shares ()
+{
+    exportfs -v |
+	grep '^/' |
+	sed -e 's@[[:space:]][[:space:]]*[^[:space:]()][^[:space:]()]*([^[:space:]()][^[:space:]()]*)$@@' |
+	sort -u
+}
+
+##################################################
+
+case "$1" in
+    shutdown)
+	nfs_shutdown
+	;;
+    startup)
+	nfs_startup
+	;;
+    stop)
+	service_stop "$2"
+	;;
+    start)
+	service_start "$2"
+	;;
+    monitor-list-shares)
+	nfs_monitor_list_shares
+	;;
+    monitor-post)
+	nfs_check_thread_count
+	;;
+    monitor-pre|releaseip|takeip)
+	# Not required/implemented
+	:
+	;;
+    *)
+	usage
+esac

ctdb/config/statd-callout

@@ -116,15 +116,15 @@ case "$1" in
 	# service is started.
 	state_even=$(( $(date '+%s') / 2 * 2))
 
-	# we must also let some time pass between stopping and restarting the
+	# We must also let some time pass between stopping and
-	# lockmanager since othervise there is a window where the lockmanager
+	# restarting the lock manager.  Otherwise there is a window
-	# will respond "strangely" immediately after restarting it, which
+	# where the lock manager will respond "strangely" immediately
-	# causes clients to fail to reclaim the locks.
+	# after restarting it, which causes clients to fail to reclaim
-	# 
+	# their locks.
 	if [ "${CTDB_NFS_SERVER_MODE:-${NFS_SERVER_MODE}}" != "ganesha" ] ; then
-            startstop_nfslock stop >/dev/null 2>&1
+	    "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
             sleep 2
-            startstop_nfslock start >/dev/null 2>&1
+	    "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
 	fi
 
 	# we now need to send out additional statd notifications to ensure

ctdb/doc/ctdbd.conf.5.xml

@@ -1055,6 +1055,21 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
 	  </listitem>
 	</varlistentry>
 
+	<varlistentry>
+	  <term>CTDB_NFS_CALLOUT=<parameter>COMMAND</parameter></term>
+	  <listitem>
+	    <para>
+	      COMMAND specifies the path to a callout to handle
+	      interactions with the configured NFS system, including
+	      startup, shutdown, monitoring.
+	    </para>
+	    <para>
+	      Default is the included
+	      <command>nfs-linux-kernel-callout</command>.
+	    </para>
+	  </listitem>
+	</varlistentry>
+
 	<varlistentry>
 	  <term>CTDB_NFS_DUMP_STUCK_THREADS=<parameter>NUM</parameter></term>
 	  <listitem>

ctdb/packaging/RPM/ctdb.spec.in

@@ -191,6 +191,7 @@ rm -rf $RPM_BUILD_ROOT
 %config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/40.mountd.check
 %config(noreplace) %{_sysconfdir}/ctdb/nfs-checks.d/50.rquotad.check
 %{_sysconfdir}/ctdb/statd-callout
+%{_sysconfdir}/ctdb/nfs-linux-kernel-callout
 %{_sbindir}/ctdbd
 %{_sbindir}/ctdbd_wrapper
 %{_bindir}/ctdb

ctdb/tests/eventscripts/60.nfs.monitor.102.sh

@@ -8,7 +8,7 @@ setup_nfs
 
 CTDB_MONITOR_NFS_THREAD_COUNT="yes"
 RPCNFSDCOUNT=8
-FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6 7 8"
+nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6 7 8
 
 ok_null
 

ctdb/tests/eventscripts/60.nfs.monitor.103.sh

@@ -8,7 +8,7 @@ setup_nfs
 
 CTDB_MONITOR_NFS_THREAD_COUNT="yes"
 RPCNFSDCOUNT=8
-FAKE_NFSD_THREAD_PIDS="1 2 3 4 5"
+nfs_setup_fake_threads "nfsd" 1 2 3 4 5
 
 ok "Attempting to correct number of nfsd threads from 5 to 8"
 

ctdb/tests/eventscripts/60.nfs.monitor.104.sh

@@ -11,7 +11,7 @@ setup_nfs
 
 CTDB_MONITOR_NFS_THREAD_COUNT="yes"
 RPCNFSDCOUNT=4
-FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6"
+nfs_setup_fake_threads "nfsd" 1 2 3 4 5 6
 
 ok "Attempting to correct number of nfsd threads from 6 to 4"
 

ctdb/tests/eventscripts/60.nfs.monitor.113.sh

@@ -10,6 +10,6 @@ rpc_services_down "nfs"
 
 # Additionally, any hung threads should have stack traces dumped.
 CTDB_NFS_DUMP_STUCK_THREADS=5
-FAKE_NFSD_THREAD_PIDS=""
+nfs_setup_fake_threads "nfsd"
 
 nfs_iterate_test 10 "nfs"

ctdb/tests/eventscripts/60.nfs.monitor.114.sh

@@ -10,6 +10,6 @@ rpc_services_down "nfs"
 
 # Additionally, any hung threads should have stack traces dumped.
 CTDB_NFS_DUMP_STUCK_THREADS=5
-FAKE_NFSD_THREAD_PIDS="1001 1002 1003"
+nfs_setup_fake_threads "nfsd" 1001 1002 1003
 
 nfs_iterate_test 10 "nfs"

ctdb/tests/eventscripts/60.nfs.monitor.143.sh

@@ -9,6 +9,6 @@ define_test "statd down, 2 iterations, stuck process"
 setup_nfs
 rpc_services_down "status"
 CTDB_NFS_DUMP_STUCK_THREADS=2
-FAKE_RPC_THREAD_PIDS="1001"
+nfs_setup_fake_threads "rpc.status" 1001
 
 nfs_iterate_test 2 "status"

ctdb/tests/eventscripts/etc-ctdb/nfs-linux-kernel-callout

@@ -0,0 +1 @@
+../../../config/nfs-linux-kernel-callout

ctdb/tests/eventscripts/scripts/local.sh

@@ -789,8 +789,8 @@ setup_nfs ()
 
     export CTDB_NFS_SKIP_SHARE_CHECK="no"
 
-    export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
+    export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT
-    export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS
+    export CTDB_NFS_DUMP_STUCK_THREADS
 
     # Reset the failcounts for nfs services.
     eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
@@ -805,6 +805,9 @@ setup_nfs ()
 	export CTDB_MANAGED_SERVICES="foo nfs bar"
 
 	rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status"
+
+	nfs_setup_fake_threads "nfsd"
+	nfs_setup_fake_threads "rpc.foobar"  # Just set the variable to empty
     else
 	debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB"
 
@@ -876,6 +879,24 @@ nfs_load_config ()
     done
 }
 
+nfs_setup_fake_threads ()
+{
+    _prog="$1" ; shift
+
+    case "$_prog" in
+	nfsd)
+	    export PROCFS_PATH=$(mktemp -d --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
+	    _threads="${PROCFS_PATH}/fs/nfsd/threads"
+	    mkdir -p $(dirname "$_threads")
+	    echo $# >"$_threads"
+	    export FAKE_NFSD_THREAD_PIDS="$*"
+	    ;;
+	*)
+	    export FAKE_RPC_THREAD_PIDS="$*"
+	    ;;
+    esac
+}
+
 program_stack_traces ()
 {
     _prog="$1"
@@ -896,10 +917,10 @@ EOF
 guess_output ()
 {
     case "$1" in
-	startstop_nfslock\ start)
+	$CTDB_NFS_CALLOUT\ start\ nlockmgr)
 	    echo "&Starting nfslock: OK"
 	    ;;
-	startstop_nfs\ start)
+	$CTDB_NFS_CALLOUT\ start\ nfs)
 	    cat <<EOF
 &Starting nfslock: OK
 &Starting nfs: OK

ctdb/wscript

@@ -486,6 +486,7 @@ def build(bld):
         'debug-hung-script.sh',
         'debug_locks.sh',
         'gcore_trace.sh',
+        'nfs-linux-kernel-callout',
         'notify.sh',
         'statd-callout'
     ]
@@ -628,6 +629,7 @@ def build(bld):
         'events.d',
         'functions',
         'nfs-checks.d',
+        'nfs-linux-kernel-callout',
         'statd-callout'
     ]