2007-09-05 08:20:34 +04:00
#!/bin/sh
# a script to test the basic setup of a CTDB/Samba install
# tridge@samba.org September 2007
2010-08-06 05:10:56 +04:00
# martin@meltin.net August 2010
usage ()
{
cat >&2 <<EOF
Usage: ctdb_diagnostics [OPTION] ...
options:
-n <nodes> Comma separated list of nodes to operate on
-c Ignore comment lines (starting with '#') in file comparisons
-w Ignore whitespace in file comparisons
--no-ads Do not use commands that assume an Active Directory Server
EOF
exit 1
}
nodes=$(ctdb listnodes -Y | cut -d: -f2)
2011-10-07 08:00:42 +04:00
bad_nodes=""
2010-08-06 05:10:56 +04:00
diff_opts=
no_ads=false
parse_options ()
{
temp=$(getopt -n "ctdb_diagnostics" -o "n:cwh" -l no-ads,help -- "$@")
[ $? != 0 ] && usage
eval set -- "$temp"
while true ; do
case "$1" in
-n) nodes=$(echo "$2" | sed -e 's@,@ @g') ; shift 2 ;;
-c) diff_opts="${diff_opts} -I ^#.*" ; shift ;;
-w) diff_opts="${diff_opts} -w" ; shift ;;
--no-ads) no_ads=true ; shift ;;
--) shift ; break ;;
-h|--help|*) usage ;;
esac
done
[ $# -ne 0 ] && usage
}
parse_options "$@"
2011-10-07 08:00:42 +04:00
# Use 5s ssh timeout if EXTRA_SSH_OPTS doesn't set a timeout.
case "$EXTRA_SSH_OPTS" in
*ConnectTimeout=*) : ;;
*)
export EXTRA_SSH_OPTS="${EXTRA_SSH_OPTS} -o ConnectTimeout=5"
esac
# Filter nodes. Remove any nodes we can't contact from $node and add
# them to $bad_nodes.
_nodes=""
for _i in $nodes ; do
if onnode $_i true >/dev/null 2>&1 ; then
_nodes="${_nodes}${_nodes:+ }${_i}"
else
bad_nodes="${bad_nodes}${bad_nodes:+,}${_i}"
fi
done
nodes="$_nodes"
2010-08-06 05:10:56 +04:00
nodes_comma=$(echo $nodes | sed -e 's@[[:space:]]@,@g')
2007-09-05 08:20:34 +04:00
PATH="$PATH:/sbin:/usr/sbin:/usr/lpp/mmfs/bin"
2007-03-10 05:45:38 +03:00
# list of config files that must exist and that we check are the same
2010-08-06 05:10:56 +04:00
# on the nodes
2011-10-15 21:50:12 +04:00
if [ -d /etc/sysconfig ] ; then
CONFIG_FILES_MUST="/etc/krb5.conf /etc/hosts /etc/ctdb/nodes /etc/sysconfig/ctdb /etc/resolv.conf /etc/nsswitch.conf /etc/sysctl.conf /etc/samba/smb.conf /etc/fstab /etc/multipath.conf /etc/pam.d/system-auth /etc/sysconfig/nfs /etc/exports /etc/vsftpd/vsftpd.conf"
else
CONFIG_FILES_MUST="/etc/krb5.conf /etc/hosts /etc/ctdb/nodes /etc/default/ctdb /etc/resolv.conf /etc/nsswitch.conf /etc/sysctl.conf /etc/samba/smb.conf /etc/fstab /etc/multipath.conf /etc/pam.d/system-auth /etc/default/nfs /etc/exports /etc/vsftpd/vsftpd.conf"
fi
2007-03-10 05:45:38 +03:00
# list of config files that may exist and should be checked that they
2010-08-06 05:10:56 +04:00
# are the same on the nodes
2007-03-10 05:45:38 +03:00
CONFIG_FILES_MAY="/etc/ctdb/public_addresses /etc/ctdb/static-routes"
2007-09-05 08:20:34 +04:00
2>&1
cat <<EOF
--------------------------------------------------------------------
ctdb_diagnostics starting. This script will gather information about
your ctdb cluster. You should send the output of this script along
with any ctdb or clustered Samba bug reports.
--------------------------------------------------------------------
EOF
date
error() {
msg="$1"
echo "ERROR: $msg"
NUM_ERRORS=`expr $NUM_ERRORS + 1`
echo " ERROR[$NUM_ERRORS]: $msg" >> $ERRORS
}
show_file() {
fname="$1"
echo " ================================"
echo " File: $fname"
2007-10-30 02:18:52 +03:00
echo " `ls -l $fname 2>&1`"
cat "$fname" 2>&1 | sed 's/^/ /'
2007-09-05 08:20:34 +04:00
echo " ================================"
}
show_all() {
2010-08-06 05:10:56 +04:00
echo "running $1 on nodes $nodes_comma"
onnode $nodes_comma "hostname; date; $1 2>&1 | sed 's/^/ /'" 2>&1
}
show_and_compare_files () {
fmt="$1" ; shift
for f ; do
first=true
for n in $nodes ; do
if $first ; then
onnode $n [ -r "$f" ] || {
msg=$(printf "$fmt" "$f" $n)
error "$msg"
continue 2;
}
2013-08-21 10:38:17 +04:00
fstf=$tmpdir/`basename $f`.node$n
2010-08-06 05:10:56 +04:00
onnode $n cat $f > $fstf 2>&1
echo " ================================"
echo " File (on node $n): $f"
echo " `onnode $n ls -l $f 2>&1`"
cat "$fstf" | sed 's/^/ /'
echo " ================================"
first=false
else
echo "Testing for same config file $f on node $n"
2013-08-21 10:38:17 +04:00
tmpf=$tmpdir/`basename $f`.node$n
2010-08-06 05:10:56 +04:00
onnode $n cat $f > $tmpf 2>&1
diff $diff_opts $fstf $tmpf >/dev/null 2>&1 || {
error "File $f is different on node $n"
diff -u $diff_opts $fstf $tmpf
}
rm -f $tmpf
fi
done
rm -f $fstf
done
2007-09-05 08:20:34 +04:00
}
2013-08-21 10:38:17 +04:00
if ! tmpdir=$(mktemp -d) ; then
echo "Unable to create a temporary directory"
exit 1
fi
ERRORS="${tmpdir}/diag_err"
2007-09-05 08:20:34 +04:00
NUM_ERRORS=0
2010-08-06 05:10:56 +04:00
2007-09-05 08:20:34 +04:00
cat <<EOF
2010-08-06 05:10:56 +04:00
Diagnosis started on these nodes:
$nodes_comma
2011-10-07 08:00:42 +04:00
EOF
if [ -n "$bad_nodes" ] ; then
cat <<EOF
NOT RUNNING DIAGNOSTICS on these uncontactable nodes:
$bad_nodes
EOF
fi
cat <<EOF
2010-08-06 05:10:56 +04:00
For reference, here is the nodes file on the current node...
2007-09-05 08:20:34 +04:00
EOF
2011-10-07 08:00:42 +04:00
show_file /etc/ctdb/nodes
2007-09-05 08:20:34 +04:00
cat <<EOF
--------------------------------------------------------------------
2010-08-06 05:10:56 +04:00
Comping critical config files on nodes $nodes_comma
2007-09-05 08:20:34 +04:00
EOF
2010-08-06 05:10:56 +04:00
show_and_compare_files \
"%s is missing on node %d" \
$CONFIG_FILES_MUST
2007-03-10 05:45:38 +03:00
2010-08-06 05:10:56 +04:00
show_and_compare_files \
"Optional file %s is not present on node %d" \
$CONFIG_FILES_MAY
2007-09-05 08:20:34 +04:00
cat <<EOF
--------------------------------------------------------------------
Checking for clock drift
EOF
t=`date +%s`
2010-08-06 05:10:56 +04:00
for i in $nodes; do
2007-09-05 08:20:34 +04:00
t2=`onnode $i date +%s`
d=`expr $t2 - $t`
if [ $d -gt 30 -o $d -lt -30 ]; then
error "time on node $i differs by $d seconds"
fi
done
cat <<EOF
--------------------------------------------------------------------
Showing software versions
EOF
show_all "uname -a"
[ -x /bin/rpm ] && {
show_all "rpm -qa | egrep 'samba|ctdb|gpfs'"
}
2008-10-13 01:21:20 +04:00
[ -x /usr/bin/dpkg-query ] && {
show_all "/usr/bin/dpkg-query --show 'ctdb'"
show_all "/usr/bin/dpkg-query --show 'samba'"
#show_all "/usr/bin/dpkg-query --show 'gpfs'"
}
2007-09-05 08:20:34 +04:00
cat <<EOF
--------------------------------------------------------------------
Showing ctdb status and recent log entries
EOF
show_all "ctdb status; ctdb ip"
show_all "ctdb statistics"
2008-09-17 15:00:04 +04:00
show_all "ctdb uptime"
2012-10-11 09:21:02 +04:00
show_all "ctdb listvars"
2013-08-21 10:48:21 +04:00
show_all "ctdb getdbmap"
2007-09-05 08:20:34 +04:00
echo "Showing log.ctdb"
2009-05-06 21:01:58 +04:00
show_all "test -f /var/log/log.ctdb && tail -100 /var/log/log.ctdb"
2007-09-05 08:20:34 +04:00
echo "Showing log.ctdb"
2009-05-06 21:01:58 +04:00
show_all "test -f /var/log/log.ctdb && tail -100 /var/log/log.ctdb"
2007-09-05 08:20:34 +04:00
2008-09-17 15:00:04 +04:00
show_all "tail -200 /var/log/messages"
show_all "tail -200 /etc/ctdb/state/vacuum.log"
show_all "ls -lRs /var/ctdb"
show_all "ls -lRs /etc/ctdb"
2007-09-05 08:20:34 +04:00
cat <<EOF
--------------------------------------------------------------------
2007-09-10 05:27:07 +04:00
Showing system and process status
2007-09-05 08:20:34 +04:00
EOF
2007-09-19 05:46:11 +04:00
show_all "df"
show_all "df -i"
show_all "mount"
2007-09-17 09:31:33 +04:00
show_all "w"
2008-09-17 15:00:04 +04:00
show_all "ps axfwu"
2007-09-05 08:20:34 +04:00
show_all "dmesg"
show_all "/sbin/lspci"
2008-09-17 15:00:04 +04:00
show_all "dmidecode"
2007-09-17 09:31:33 +04:00
show_all "cat /proc/partitions"
show_all "cat /proc/cpuinfo"
show_all "cat /proc/scsi/scsi"
2007-09-05 08:20:34 +04:00
show_all "/sbin/ifconfig -a"
2008-09-17 15:00:04 +04:00
show_all "/sbin/ifconfig -a"
2007-09-05 08:20:34 +04:00
show_all "/sbin/ip addr list"
show_all "/sbin/route -n"
2007-09-17 09:31:33 +04:00
show_all "netstat -s"
2007-10-30 02:18:52 +03:00
show_all "free"
2007-09-10 05:27:07 +04:00
show_all "crontab -l"
show_all "sysctl -a"
2009-10-06 18:16:13 +04:00
show_all "iptables -L -n"
show_all "iptables -L -n -t nat"
2007-09-17 09:31:33 +04:00
show_all "/usr/sbin/rpcinfo -p"
show_all "/usr/sbin/showmount -a"
show_all "/usr/sbin/showmount -e"
show_all "/usr/sbin/nfsstat -v"
[ -x /sbin/multipath ] && {
2007-10-16 14:13:28 +04:00
show_all "/sbin/multipath -ll"
2007-09-17 09:31:33 +04:00
}
[ -x /sbin/chkconfig ] && {
show_all "/sbin/chkconfig --list"
}
[ -x /usr/sbin/getenforce ] && {
show_all "/usr/sbin/getenforce"
}
2007-10-30 02:18:52 +03:00
[ -d /proc/net/bonding ] && {
for f in /proc/net/bonding/*; do
show_all "cat $f"
done
}
2007-09-05 08:20:34 +04:00
cat <<EOF
--------------------------------------------------------------------
Showing Samba status
EOF
show_all "smbstatus -n -B"
2010-08-06 05:10:56 +04:00
if $no_ads ; then
echo
echo "Skipping \"net ads testjoin\" as requested"
echo
else
show_all "net ads testjoin"
fi
2007-03-10 06:10:21 +03:00
show_all "net conf list"
2007-09-05 08:20:34 +04:00
show_all "lsof -n | grep smbd"
2008-09-17 15:00:04 +04:00
show_all "lsof -n | grep ctdbd"
2007-09-05 08:20:34 +04:00
show_all "netstat -tan"
2010-08-06 05:10:56 +04:00
if $no_ads ; then
echo
echo "Skipping \"net ads info\" as requested"
echo
else
show_all "net ads info"
fi
2007-09-05 08:20:34 +04:00
show_all "date"
show_all "smbclient -U% -L 127.0.0.1"
WORKGROUP=`testparm -s --parameter-name=WORKGROUP 2> /dev/null`
show_all id "$WORKGROUP/Administrator"
show_all "wbinfo -p"
2009-03-19 02:43:57 +03:00
show_all "wbinfo --online-status"
2007-09-05 08:20:34 +04:00
show_all "smbd -b"
date
echo "Diagnostics finished with $NUM_ERRORS errors"
[ -r $ERRORS ] && {
cat $ERRORS
rm -f $ERRORS
}
2013-08-21 10:38:17 +04:00
rm -rf "$tmpdir"
2007-09-05 08:20:34 +04:00
exit $NUM_ERRORS