sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2025-01-21 18:04:06 +03:00
Code

ctdb-scripts: Use nfs-utils' sm-notify instead of CTDB's smnotify

Browse Source 
CTDB's smnotify does not support IPv6 and is difficult to maintain.

So, create directories of files and pass them to NFS util's sm-notify.

There is an implied change here, because NFS utils sm-notify stopped
sending IP addresses as mon_name back in 2010:

  http://git.linux-nfs.org/?p=steved/nfs-utils.git;a=commitdiff;h=900df0e7c0b9006d72d8459b30dc2cd69ce495a5

This will change advice given in the wiki to use a hostname for the
cluster with round-robin DNS, since this is what is best supported.

Another behavioural change is that sm-notify only sends "up"
notifications with an odd state.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Signed-off-by: Martin Schwenke <mschwenke@ddn.com>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
This commit is contained in:
Martin Schwenke 2017-03-03 15:44:08 +11:00 committed by Martin Schwenke
parent d89506449f
commit ece6153038
9 changed files with 183 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ctdb/.gitignore vendored
View File

@ -14,9 +14,6 @@ Makefile
config.h
config.h.in
config.log
utils/smnotify/gen_smnotify.c
utils/smnotify/gen_xdr.c
utils/smnotify/smnotify.h
nodes.txt
public_addresses.txt
rec.lock

16
ctdb/tests/UNIT/eventscripts/scripts/statd-callout.sh
View File

@ -3,6 +3,8 @@ setup()
setup_public_addresses
ctdb_set_pnn
setup_date "1234567890"
export NFS_HOSTNAME
}
ctdb_catdb_format_pairs()
@ -48,8 +50,13 @@ EOF
check_statd_callout_smnotify()
{
_state_even=$(( $(date '+%s') / 2 * 2))
_state_odd=$((_state_even + 1))
# The state here doesn't really matter because the date stub
# generates a fixed value (as per above setup() function,
# which happens to set it to an even value). In reality,
# sm-notify would convert it to an odd value, but for testing
# it doesn't really matter because the sm-notify stub just
# prints the state and it just needs to be matched.
_state=$(date '+%s')
nfs_load_config
@ -57,10 +64,7 @@ check_statd_callout_smnotify()
while read -r _ _sip _; do
for _cip; do
cat <<EOF
SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${_sip}, STATE=${_state_even}
SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${_state_even}
SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${_sip}, STATE=${_state_odd}
SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${_state_odd}
SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${_state}
EOF
done
done | {

44
ctdb/tests/UNIT/eventscripts/stubs/ctdb_smnotify_helper Executable file
View File

@ -0,0 +1,44 @@
#!/bin/sh
print_monitor_line()
{
_client_ip="$1"
_server_ip="$2"
_priv="00000000000000000000000000000000"
# Pass 0 for local IP instead of doing endianness conversion
printf '%08x %08x %08x %08x %s %s %s\n' \
0 \
100021 \
4 \
16 \
"$_priv" \
"$_client_ip" \
"$_server_ip"
}
print_state()
{
_state="$1"
# Write the state as a host order 32-bit integer
_imports="import sys; import struct;"
_expr="sys.stdout.buffer.write(struct.pack('i', int(${_state})))"
python3 -c "${_imports} ${_expr}"
}
usage()
{
printf 'usage: %s { monitor <client-ip> <source-ip> | state <stat> }\n' \
"$1"
exit 1
}
if [ $# -eq 3 ] && [ "$1" = "monitor" ]; then
print_monitor_line "$2" "$3"
elif [ $# -eq 2 ] && [ "$1" = "state" ]; then
print_state "$2"
else
usage "$0"
fi

76
ctdb/tests/UNIT/eventscripts/stubs/sm-notify Executable file
View File

@ -0,0 +1,76 @@
#!/bin/sh
usage()
{
_prog="${0##*/}" # basename
cat <<EOF
Usage: ${_prog} [-dfn] [-m MINUTES] [-P PATH] [-v name]
EOF
exit 1
}
temp=$(getopt -n "sm-notify" -o "dfnm:P:v:h" -l "help" -- "$@")
# shellcheck disable=SC2181
# Would create unreadable long line
if [ $? != 0 ] ; then
usage
fi
eval set -- "$temp"
no_detach=false
force=false
minutes=15
no_update_state=false
path=""
mon_name=""
while : ; do
case "$1" in
-d) no_detach=true ; shift ;;
-f) force=true ; shift ;;
-n) no_update_state=true ; shift ;;
-m) minutes="$2" ; shift 2 ;;
-P) path="$2" ; shift 2 ;;
-v) mon_name="$2" ; shift 2 ;;
--) shift ; break ;;
*) usage ;;
esac
done
# Silence shellcheck regarding unused variables, which serve to
# document the possible options, which might be used in the future
: "$force" "$no_update_state" "$minutes"
if ! $no_detach ; then
echo "Not supported without -P"
usage
fi
if [ -z "$path" ] ; then
echo "Not supported without -P"
usage
fi
if [ -z "$mon_name" ] ; then
echo "Not supported without -v"
usage
fi
read_state_file ()
{
_path="$1"
od -t d4 "${_path}/state" | sed -n -e 's|^00* *||p'
}
state=$(read_state_file "$path")
find "${path}/sm" -type f |
sort |
while IFS="" read -r file ; do
read -r _ _ _ _ _ cip sip <"$file"
cat <<EOF
SM_NOTIFY: ${sip} -> ${cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${state}
EOF
done

65
ctdb/tests/UNIT/eventscripts/stubs/smnotify
View File

@ -1,65 +0,0 @@
#!/bin/sh
usage()
{
_prog="${0##*/}" # basename
cat <<EOF
Usage: ${_prog} --client=CLIENT --ip=IP --server=SERVER --stateval=STATEVAL
EOF
exit 1
}
cip=""
sip=""
mon_name=""
state=""
while [ $# -gt 0 ]; do
case "$1" in
--client)
cip="$2"
shift 2
;;
--client=*)
cip="${1#*=}"
shift
;;
--ip)
sip="$2"
shift 2
;;
--ip=*)
sip="${1#*=}"
shift
;;
--server)
mon_name="$2"
shift 2
;;
--server=*)
mon_name="${1#*=}"
shift
;;
--stateval)
state="$2"
shift 2
;;
--stateval=*)
state="${1#*=}"
shift
;;
--)
shift
break
;;
-*) usage ;;
*) break ;;
esac
done
[ $# -eq 0 ] || usage
if [ -z "$cip" ] || [ -z "$sip" ] || [ -z "$mon_name" ] || [ -z "$state" ]; then
usage
fi
echo "SM_NOTIFY: ${sip} -> ${cip}, MON_NAME=${mon_name}, STATE=${state}"

86
ctdb/tools/statd_callout_helper
View File

@ -14,6 +14,16 @@
# NFS_HOSTNAME=mycluster
# STATD_HOSTNAME="${NFS_HOSTNAME} -H /usr/local/libexec/ctdb/statd_callout"
#
# If using Linux kernel NFS then the following should also be set in
# /etc/nfs.conf:
#
# [sm-notify]
# lift-grace = n
#
# See sm-notify(8) for details. This doesn't matter when using
# NFS-Ganesha because sm-notify's attempt to lift grace will fail
# silently if /proc/fs/lockd/nlm_end_grace is not found.
#
if [ -z "$CTDB_BASE" ] ; then
export CTDB_BASE="/usr/local/etc/ctdb"
@ -90,41 +100,59 @@ statd_callout_queue_dir="${statd_callout_state_dir}/queue"
############################################################
# Read pairs of:
# server-IP client-IP
# from stdin and send associated SM_NOTIFY packets.
send_notifies()
{
_smnotify="${CTDB_HELPER_BINDIR}/smnotify"
# State must monotonically increase, across the entire
# cluster. Use seconds since epoch and hope the time is in
# cluster. Use seconds since epoch and assume the time is in
# sync across nodes. Even numbers mean service is shut down,
# odd numbers mean service is started.
# odd numbers mean service is up. However, sm-notify always
# reads the state and converts it to odd (if necessary, by
# adding 1 when it is even) because it only sends "up"
# notifications. Note that there is a 2038 issue here but we
# will get to that later.
_state=$(date '+%s')
# Intentionally round to an even number
# shellcheck disable=SC2017
_state_even=$(($(date '+%s') / 2 * 2))
_helper="${CTDB_HELPER_BINDIR}/ctdb_smnotify_helper"
_notify_dir="${statd_callout_state_dir}/sm-notify"
mkdir -p "$_notify_dir"
_prev=""
while read -r _sip _cip; do
# NOTE: Consider optimising smnotify to read all the
# data from stdin and then run it in the background.
# Create a directory per server IP containing a file
# for each client IP
mkdir -p \
"${_notify_dir}/${_sip}/sm" \
"${_notify_dir}/${_sip}/sm.bak"
# Reset stateval for each serverip
if [ "$_sip" != "$_prev" ]; then
_stateval="$_state_even"
_out="${_notify_dir}/${_sip}/sm/${_cip}"
"$_helper" "monitor" "$_cip" "$_sip" >"$_out"
done
# Send notifications for server startup
_ref=$(find_statd_sm_dir)
for _sip_dir in "$_notify_dir"/*; do
if [ "$_sip_dir" = "${_notify_dir}/*" ]; then
break
fi
# Send notifies for server shutdown
"$_smnotify" --client="$_cip" --ip="$_sip" \
--server="$_sip" --stateval="$_stateval"
"$_smnotify" --client="$_cip" --ip="$_sip" \
--server="$NFS_HOSTNAME" --stateval="$_stateval"
_sip="${_sip_dir##*/}" # basename
# Send notifies for server startup
_stateval=$((_stateval + 1))
"$_smnotify" --client="$_cip" --ip="$_sip" \
--server="$_sip" --stateval="$_stateval"
"$_smnotify" --client="$_cip" --ip="$_sip" \
--server="$NFS_HOSTNAME" --stateval="$_stateval"
# Write the state as a host order 32-bit integer. See
# note at top of function about state.
_out="${_sip_dir}/state"
"$_helper" "state" "$_state" >"$_out"
# The ownership of the directory and contents should
# match the system's statd sm directory, so that
# sm-notify drops privileges and switches to run as
# the directory owner.
chown -R --reference="$_ref" "$_sip_dir"
timeout 10 sm-notify -d -f -m 0 -n -P "$_sip_dir" -v "$_sip"
rm -rf "$_sip_dir"
done
}
@ -229,21 +257,13 @@ notify)
# clients are ok with, buth other clients will barf at.
# 2, Some clients only accept statd packets IFF they come from the
# 'correct' ip address.
# 2a,Send out the notification using the 'correct' ip address and also
# Send out the notification using the 'correct' ip address and also
# specify the 'correct' hostname in the statd packet.
# Some clients require both the correct source address and also the
# correct name. (these clients also ONLY work if the ip addresses
# used to map the share can be resolved into the name returned in
# the notify packet.)
# 2b,Other clients require that the source ip address of the notify
# packet matches the ip address used to take out the lock.
# I.e. that the correct source address is used.
# These clients also require that the statd notify packet contains
# the name as the ip address used when the lock was taken out.
#
# Both 2a and 2b are commonly used in lockmanagers since they maximize
# probability that the client will accept the statd notify packet and
# not just ignore it.
# For all IPs we serve, collect info and push to the config database
# Construct a sed expression to take catdb output and produce pairs of:

151
ctdb/utils/smnotify/smnotify.c
View File

@ -1,151 +0,0 @@
/*
simple smnotify tool
Copyright (C) Ronnie Sahlberg 2007
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include "smnotify.h"
#include "popt.h"
static char *client = NULL;
static const char *ip = NULL;
static char *server = NULL;
static int stateval = 0;
static int clientport = 0;
static int sendport = 0;
static void usage(void)
{
exit(0);
}
static int create_socket(const char *addr, int port)
{
int s;
struct sockaddr_in sock_in;
s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (s == -1) {
printf("Failed to open local socket\n");
exit(10);
}
bzero(&sock_in, sizeof(sock_in));
sock_in.sin_family = AF_INET;
sock_in.sin_port = htons(port);
inet_aton(addr, &sock_in.sin_addr);
if (bind(s, (struct sockaddr *)&sock_in, sizeof(sock_in)) == -1) {
printf("Failed to bind to local socket\n");
exit(10);
}
return s;
}
int main(int argc, const char *argv[])
{
struct poptOption popt_options[] = {
POPT_AUTOHELP
{ "client", 'c', POPT_ARG_STRING, &client, 0, "remote client to send the notify to", "hostname/ip" },
{ "clientport", 0, POPT_ARG_INT, &clientport, 0, "clientport", "integer" },
{ "ip", 'i', POPT_ARG_STRING, &ip, 0, "local ip address to send the notification from", "ip" },
{ "sendport", 0, POPT_ARG_INT, &sendport, 0, "port to send the notify from", "integer" },
{ "server", 's', POPT_ARG_STRING, &server, 0, "servername to use in the notification", "hostname/ip" },
{ "stateval", 0, POPT_ARG_INT, &stateval, 0, "stateval", "integer" },
POPT_TABLEEND
};
int opt;
poptContext pc;
CLIENT *clnt;
int s;
struct sockaddr_in sock_cl;
struct timeval w;
struct status st;
pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
while ((opt = poptGetNextOpt(pc)) != -1) {
switch (opt) {
default:
fprintf(stderr, "Invalid option %s: %s\n",
poptBadOption(pc, 0), poptStrerror(opt));
exit(1);
}
}
if (client == NULL) {
printf("ERROR: client not specified\n");
usage();
}
if (ip == NULL) {
printf("ERROR: ip not specified\n");
usage();
}
if (server == NULL) {
printf("ERROR: server not specified\n");
usage();
}
if (stateval == 0) {
printf("ERROR: stateval not specified\n");
usage();
}
/* Since we want to control from which address these packets are
sent we must create the socket ourself and use low-level rpc
calls.
*/
s = create_socket(ip, sendport);
/* only wait for at most 3 seconds before giving up */
alarm(3);
/* Setup a sockaddr_in for the client we want to notify */
bzero(&sock_cl, sizeof(sock_cl));
sock_cl.sin_family = AF_INET;
sock_cl.sin_port = htons(clientport);
inet_aton(client, &sock_cl.sin_addr);
w.tv_sec = 1;
w.tv_usec= 0;
clnt = clntudp_create(&sock_cl, 100024, 1, w, &s);
if (clnt == NULL) {
printf("ERROR: failed to connect to client\n");
exit(10);
}
/* we don't want to wait for any reply */
w.tv_sec = 0;
w.tv_usec = 0;
clnt_control(clnt, CLSET_TIMEOUT, (char *)&w);
st.mon_name=server;
st.state=stateval;
sm_notify_1(&st, clnt);
return 0;
}

21
ctdb/utils/smnotify/smnotify.x
View File

@ -1,21 +0,0 @@
#ifdef RPC_HDR
%#ifdef _AIX
%#include <rpc/rpc.h>
%#endif /* _AIX */
#endif /* RPC_HDR */
const SM_MAXSTRLEN = 1024;
struct status {
string mon_name<SM_MAXSTRLEN>;
int state;
};
program SMNOTIFY {
version SMVERSION {
void SM_NOTIFY(struct status) = 6;
} = 1;
} = 100024;

24
ctdb/wscript
View File

@ -690,30 +690,6 @@ def build(bld):
deps='sys_rw replace',
install_path='${CTDB_HELPER_BINDIR}')
bld.SAMBA_GENERATOR('ctdb-smnotify-h',
source='utils/smnotify/smnotify.x',
target='utils/smnotify/smnotify.h',
rule='rpcgen -h ${SRC} > ${TGT}')
xdr_buf_hack = 'grep -Fv "register int32_t *buf;"'
bld.SAMBA_GENERATOR('ctdb-smnotify-x',
source='utils/smnotify/smnotify.x',
target='utils/smnotify/gen_xdr.c',
rule='rpcgen -c ${SRC} | ' + xdr_buf_hack + ' > ${TGT}')
bld.SAMBA_GENERATOR('ctdb-smnotify-c',
source='utils/smnotify/smnotify.x',
target='utils/smnotify/gen_smnotify.c',
rule='rpcgen -l ${SRC} > ${TGT}')
bld.SAMBA_BINARY('smnotify',
source=bld.SUBDIR('utils/smnotify',
'smnotify.c gen_smnotify.c gen_xdr.c'),
deps='ctdb-smnotify-h ctdb-smnotify-c ctdb-smnotify-x popt tirpc',
includes='utils utils/smnotify',
install_path='${CTDB_HELPER_BINDIR}')
bld.SAMBA_BINARY('ping_pong',
source='utils/ping_pong/ping_pong.c',
deps='',