From e704eb580a503098d0030b6d7e68d98bd11153ae Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 6 Jul 2023 08:20:37 +1000 Subject: [PATCH] ctdb-scripts: Use CTDB_NFS_SHARED_STATE_DIR in nfs-ganesha-callout Rename CTDB_NFS_STATE_MNT to CTDB_NFS_SHARED_STATE_DIR. It doesn't have to be a mount but can be any directory in a cluster filesystem. CTDB_NFS_SHARED_STATE_DIR will soon be used in statd_callout_helper, so the variable name might as well be better. With this change, it will still only be used by nfs-ganesha-callout, which isn't yet supported (i.e. it still lives in doc/examples). The rest of the comments below refer to behaviour changes in that script. CTDB_NFS_SHARED_STATE_DIR is now mandatory when GPFS is used. This is much saner that choosing the first GPFS filesystem - if the state directory changes then connection metadata can be lost. Drop CTDB_NFS_STATE_FS_TYPE. The filesystem type is now determined from CTDB_NFS_SHARED_STATE_DIR and it is now checked against supported filesystems. This will catch the case when the filesystem for the specified directory has not been mounted and the filesystem for the mountpoint (e.g. ext4) is not a supported filesystem for shared state. A side-effect is that the filesystem containing CTDB_NFS_SHARED_STATE_DIR must be mounted when nfs-ganesha-callout is first run. While touching this file, my shfmt pre-commit hook wants to insert a trailing ;; into a case statement. Let's sneak that in here too. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs --- ctdb/config/functions | 7 +-- ctdb/doc/ctdb-script.options.5.xml | 37 ++++++---------- ctdb/doc/examples/config_migrate.sh | 3 +- ctdb/doc/examples/nfs-ganesha-callout | 63 +++++++++++++++++---------- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/ctdb/config/functions b/ctdb/config/functions index 1ca3cebbbca..1d80c61e5f2 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -1160,11 +1160,8 @@ nfs_callout_init() export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state" # Export, if set, for use by clustered NFS callouts - if [ -n "$CTDB_NFS_STATE_FS_TYPE" ]; then - export CTDB_NFS_STATE_FS_TYPE - fi - if [ -n "$CTDB_NFS_STATE_MNT" ]; then - export CTDB_NFS_STATE_MNT + if [ -n "$CTDB_NFS_SHARED_STATE_DIR" ]; then + export CTDB_NFS_SHARED_STATE_DIR fi if [ -n "$CTDB_NFS_EXPORTS_FILE" ]; then export CTDB_NFS_EXPORTS_FILE diff --git a/ctdb/doc/ctdb-script.options.5.xml b/ctdb/doc/ctdb-script.options.5.xml index 9298f9f3498..6df82227989 100644 --- a/ctdb/doc/ctdb-script.options.5.xml +++ b/ctdb/doc/ctdb-script.options.5.xml @@ -975,6 +975,19 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000 + + + CTDB_NFS_SHARED_STATE_DIR=DIRECTORY + + + + DIRECTORY where clustered NFS shared state will be + located. DIRECTORY should be in a cluster filesystem + that is shared between the nodes. No default. + + + + CTDB_NFS_SKIP_SHARE_CHECK=yes|no @@ -1028,30 +1041,6 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000 - - - CTDB_NFS_STATE_FS_TYPE=TYPE - - - - The type of filesystem used for a clustered NFS' shared - state. No default. - - - - - - - CTDB_NFS_STATE_MNT=DIR - - - - The directory where a clustered NFS' shared state will be - located. No default. - - - - diff --git a/ctdb/doc/examples/config_migrate.sh b/ctdb/doc/examples/config_migrate.sh index 874e96c880f..5d1f7a2084b 100755 --- a/ctdb/doc/examples/config_migrate.sh +++ b/ctdb/doc/examples/config_migrate.sh @@ -255,8 +255,7 @@ CTDB_NFS_CHECKS_DIR CTDB_NFS_SKIP_SHARE_CHECK CTDB_RPCINFO_LOCALHOST CTDB_RPCINFO_LOCALHOST6 -CTDB_NFS_STATE_FS_TYPE -CTDB_NFS_STATE_MNT +CTDB_NFS_SHARED_STATE_DIR # 70.iscsi CTDB_START_ISCSI_SCRIPTS # 00.ctdb diff --git a/ctdb/doc/examples/nfs-ganesha-callout b/ctdb/doc/examples/nfs-ganesha-callout index 74c3ad5c1f9..5acd2354a96 100755 --- a/ctdb/doc/examples/nfs-ganesha-callout +++ b/ctdb/doc/examples/nfs-ganesha-callout @@ -36,10 +36,40 @@ # Exit on 1st error set -e -# Filesystem type and mount point for the (typically clustered) -# volume that will contain the NFS-Ganesha state. -state_fs="${CTDB_NFS_STATE_FS_TYPE:-gpfs}" -state_dir="${CTDB_NFS_STATE_MNT}" # No sane default. +die() +{ + echo "$1" + exit 1 +} + +# Shared directory, typically on a cluster filesystem, that will +# contain the NFS-Ganesha state +if [ -z "$CTDB_NFS_SHARED_STATE_DIR" ]; then + die "$0: CTDB_NFS_SHARED_STATE_DIR is not set" +fi + +if [ ! -d "$CTDB_NFS_SHARED_STATE_DIR" ]; then + t="$CTDB_NFS_SHARED_STATE_DIR" # Readability, below + die "$0: CTDB_NFS_SHARED_STATE_DIR=${t} not found" +fi + +state_fs=$(findmnt -n --target "$CTDB_NFS_SHARED_STATE_DIR" -o FSTYPE) + +case "$state_fs" in +glusterfs | gpfs) + : + ;; +fuse.glusterfs) + state_fs="glusterfs" + ;; +*) + d="$CTDB_NFS_SHARED_STATE_DIR" + die "$0: filesystem type \"${state_fs}\" is not supported for ${d}" + ;; +esac + +# Always put NFS-Ganesha state in its own subdirectory +state_dir="${CTDB_NFS_SHARED_STATE_DIR}/ganesha" # Location of exports file nfs_exports_file="${CTDB_NFS_EXPORTS_FILE:-/etc/ganesha/ganesha.conf}" @@ -47,7 +77,6 @@ nfs_exports_file="${CTDB_NFS_EXPORTS_FILE:-/etc/ganesha/ganesha.conf}" # To change the following, edit the default values below. Do not set # these - they aren't configuration variables, just hooks for testing. nfs_service="${CTDB_NFS_SERVICE:-nfs-ganesha}" -ganesha_rec_subdir=${CTDB_GANESHA_REC_SUBDIR:-.ganesha} procfs=${PROCFS_PATH:-/proc} case "$state_fs" in @@ -55,13 +84,9 @@ gpfs) GANRECDIR="/var/lib/nfs/ganesha" ;; glusterfs) - if [ -z "${state_dir}" ]; then - echo "CTDB_NFS_STATE_MNT not defined for GlusterFS" - exit 1 - fi host=$(hostname) - NODESTATEDIR="$state_dir/nfs-ganesha/$host" - GANSTATEDIR="$state_dir/nfs-ganesha/.noderefs" + NODESTATEDIR="${state_dir}/${host}" + GANSTATEDIR="${state_dir}/.noderefs" NODESTATELN="$GANSTATEDIR/$host" ;; esac @@ -184,21 +209,10 @@ get_cluster_fs_state() create_ganesha_recdirs() { - if ! _mounts=$(mount | grep "$state_fs"); then - echo "Failed to find mounts of type $state_fs" - exit 1 - fi - if [ -z "$_mounts" ]; then - echo "startup $state_fs not ready" - exit 0 - fi - case "$state_fs" in gpfs) - _mntpt=$(echo "$_mounts" | sort | awk 'NR == 1 {print $3}') - _link_dst="${_mntpt}/${ganesha_rec_subdir}" - mkdir -vp "$_link_dst" - check_ln "$_link_dst" "$GANRECDIR" + mkdir -vp "$state_dir" + check_ln "$state_dir" "$GANRECDIR" ;; glusterfs) [ -d /var/lib/nfs.backup ] || @@ -279,6 +293,7 @@ nfs_stats() # unhelpful for avoiding an unhealthy service echo "Not implemented" >&2 exit 1 + ;; esac }