ctdb-tests: Avoid bulk output in $out, prefer $outfile

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

ctdb/tests/complex/11_ctdb_delip_removes_ip.sh

@@ -22,8 +22,8 @@ cluster_is_healthy
 select_test_node_and_ips
 get_test_ip_mask_and_iface
 
-echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..."
-try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet6?[[:space:]]*${test_ip}/'"
+echo "Checking that node ${test_node} hosts ${test_ip}..."
+try_command_on_node $test_node "ip addr show to ${test_ip} | grep -q ."
 
 echo "Attempting to remove ${test_ip} from node ${test_node}."
 try_command_on_node $test_node $CTDB delip $test_ip
@@ -33,10 +33,10 @@ wait_until_ips_are_on_node '!' $test_node $test_ip
 timeout=60
 increment=5
 count=0
-echo "Waiting for ${test_ip} to disappear from ${iface}..."
+echo "Waiting for ${test_ip} to disappear from node ${test_node}..."
 while : ; do
-    try_command_on_node -v $test_node "ip addr show dev $iface"
-    if echo "$out" | grep -E 'inet6?[[:space:]]*${test_ip}/'; then
+    try_command_on_node -v $test_node "ip addr show to ${test_node}"
+    if -n "$out" ; then
 	echo "Still there..."
 	if [ $(($count * $increment)) -ge $timeout ] ; then
 	    echo "BAD: Timed out waiting..."

ctdb/tests/complex/18_ctdb_reloadips.sh

@@ -48,12 +48,12 @@ select_test_node_and_ips
 
 echo "Getting public IP information from CTDB..."
 try_command_on_node any "$CTDB ip -X -v all"
-ctdb_ip_info=$(echo "$out" | awk -F'|' 'NR > 1 { print $2, $3, $5 }')
+ctdb_ip_info=$(awk -F'|' 'NR > 1 { print $2, $3, $5 }' "$outfile")
 
 echo "Getting IP information from interfaces..."
 try_command_on_node all "ip addr show"
-ip_addr_info=$(echo "$out" | \
-    awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }')
+ip_addr_info=$(awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }' \
+		   "$outfile")
 
 prefix=""
 for b in $(seq 0 255) ; do
@@ -168,7 +168,7 @@ check_ips ()
 
     try_command_on_node $test_node "ip addr show dev ${iface}"
     local ip_addrs_file=$(mktemp)
-    echo "$out" | \
+    cat "$outfile" | \
 	sed -n -e "s@.*inet * \(${prefix//./\.}\.[0-9]*\)/.*@\1@p" | \
 	sort >"$ip_addrs_file"
 

ctdb/tests/complex/32_cifs_tickle.sh

@@ -61,13 +61,6 @@ echo "Source socket is $src_socket"
 # we sometimes beat the registration.
 echo "Checking if CIFS connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    die "BAD: Socket not tracked by CTDB."
-fi
 
 # This is almost immediate.  However, it is sent between nodes
 # asynchonously, so it is worth checking...

ctdb/tests/complex/36_smb_reset_server.sh

@@ -59,16 +59,8 @@ echo "Source socket is $src_socket"
 
 # This should happen as soon as connection is up... but unless we wait
 # we sometimes beat the registration.
-echo "Checking if CIFS connection is tracked by CTDB on test node..."
+echo "Waiting until SMB connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    echo "BAD: Socket not tracked by CTDB."
-    exit 1
-fi
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -80,7 +72,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node

ctdb/tests/complex/37_nfs_reset_server.sh

@@ -60,7 +60,7 @@ echo "Source socket is $src_socket"
 echo "Wait until NFS connection is tracked by CTDB on test node ..."
 wait_until $((monitor_interval * 2)) \
 	   check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
+cat "$outfile"
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -72,7 +72,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node

ctdb/tests/complex/60_rogueip_releaseip.sh

@@ -31,7 +31,7 @@ for i in $all_pnns ; do
 		continue
 	fi
 	try_command_on_node $i "$CTDB ip"
-	n=$(awk -v ip="$test_ip" '$1 == ip { print }' <<<"$out")
+	n=$(awk -v ip="$test_ip" '$1 == ip { print }' "$outfile")
 	if [ -n "$n" ] ; then
 		other_node="$i"
 		break

ctdb/tests/complex/scripts/local.bash

@@ -67,7 +67,7 @@ check_tickles ()
     local src_socket="$4"
     try_command_on_node $node ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    [ "${out/SRC: ${src_socket} /}" != "$out" ]
+    grep -Fq "SRC: ${src_socket} " "$outfile"
 }
 
 check_tickles_all ()
@@ -79,8 +79,7 @@ check_tickles_all ()
 
     try_command_on_node all ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    local t="${src_socket//./\\.}"
-    local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
+    local count=$(grep -Fc "SRC: ${src_socket} " "$outfile" || true)
     [ $count -eq $numnodes ]
 }
 

ctdb/tests/scripts/integration.bash

@@ -148,30 +148,7 @@ sanity_check_output ()
     return $ret
 }
 
-sanity_check_ips ()
-{
-    local ips="$1" # list of "ip node" lines
-
-    echo "Sanity checking IPs..."
-
-    local x ipp prev
-    prev=""
-    while read x ipp ; do
-	[ "$ipp" = "-1" ] && break
-	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
-	    echo "OK"
-	    return 0
-	fi
-	prev="$ipp"
-    done <<<"$ips"
-
-    echo "BAD: a node was -1 or IPs are only assigned to one node:"
-    echo "$ips"
-    echo "Are you running an old version of CTDB?"
-    return 1
-}
-
-# This returns a list of "ip node" lines in $out
+# This returns a list of "ip node" lines in $outfile
 all_ips_on_node()
 {
     local node="$1"
@@ -192,9 +169,9 @@ _select_test_node_and_ips ()
 	    test_node="$pnn"
 	fi
 	if [ "$pnn" = "$test_node" ] ; then
-            test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+	    test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
     test_ip="${test_node_ips%% *}"
@@ -264,7 +241,7 @@ delete_ip_from_all_nodes ()
 	    if [ "$_ip" = "$_i" ] ; then
 		_nodes="${_nodes}${_nodes:+,}${_pnn}"
 	    fi
-	done <<<"$out" # bashism
+	done <"$outfile"
     done
 
     try_command_on_node -pq "$_nodes" "$CTDB delip $_ip"
@@ -435,7 +412,7 @@ ips_are_on_node ()
 	    if $negating ; then
 		ips="${ips/${check}}"
 	    fi
-	done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+	done <"$outfile"
     done
 
     ips="${ips// }" # Remove any spaces.
@@ -477,7 +454,7 @@ node_has_some_ips ()
 	if [ "$node" = "$pnn" ] ; then
 	    return 0
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     return 1
 }
@@ -614,7 +591,8 @@ wait_for_monitor_event ()
 	return 1
     }
 
-    local ctdb_scriptstatus_original="$out"
+    mv "$outfile" "${outfile}.orig"
+
     wait_until 120 _ctdb_scriptstatus_changed
 }
 
@@ -625,7 +603,7 @@ _ctdb_scriptstatus_changed ()
 	return 1
     }
 
-    [ "$out" != "$ctdb_scriptstatus_original" ]
+    ! diff "$outfile" "${outfile}.orig" >/dev/null
 }
 
 #######################################

ctdb/tests/simple/02_ctdb_tunables.sh

@@ -23,7 +23,6 @@ sanity_check_output \
 
 echo "Verifying all variable values using \"ctdb getvar\"..."
 
-echo "$out" |
 while read var x val ; do
     try_command_on_node 0 "$CTDB getvar $var"
 
@@ -33,7 +32,7 @@ while read var x val ; do
 	echo "MISMATCH on $var: $val != $val2"
 	exit 1
     fi
-done
+done <"$outfile"
 
 echo "GOOD: all tunables match"
 

ctdb/tests/simple/05_ctdb_listnodes.sh

@@ -32,7 +32,7 @@ cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listnodes"
 
-num_nodes=$(echo "$out" | wc -l)
+num_nodes=$(wc -l <"$outfile")
 
 # Each line should look like an IP address.
 ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'

ctdb/tests/simple/08_ctdb_isnotrecmaster.sh

@@ -33,13 +33,11 @@ set -e
 cluster_is_healthy
 
 cmd="$CTDB isnotrecmaster || true"
-try_command_on_node all "$cmd"
-echo "Output of \"$cmd\":"
-echo "$out"
+try_command_on_node -v all "$cmd"
 
-num_all_lines=$(echo "$out" |  wc -l)
-num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
-num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
+num_all_lines=$(wc -l <"$outfile")
+num_rm_lines=$(grep -Fc 'this node is the recmaster' "$outfile") || true
+num_not_rm_lines=$(grep -Fc 'this node is not the recmaster' "$outfile") || true
 
 if [ $num_rm_lines -eq 1 ] ; then
     echo "OK, there is only 1 recmaster"

ctdb/tests/simple/11_ctdb_ip.sh

@@ -33,18 +33,20 @@ cluster_is_healthy
 
 echo "Getting list of public IPs..."
 try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
-ips=$(echo "$out" | sed \
+ips=$(sed \
 	-e 's@ node\[@ @' \
-	-e 's@\].*$@@')
-machineout=$(echo "$out" | sed -r \
+	-e 's@\].*$@@' \
+	"$outfile")
+machineout=$(sed -r \
 	-e 's@^| |$@\|@g' \
 	-e 's@[[:alpha:]]+\[@@g' \
-	-e 's@\]@@g')
+	-e 's@\]@@g' \
+	"$outfile")
 
 if [ -z "$TEST_LOCAL_DAEMONS" ]; then
 	while read ip pnn ; do
-		try_command_on_node $pnn "ip addr show"
-		if [ "${out/inet* ${ip}\/}" != "$out" ] ; then
+		try_command_on_node $pnn "ip addr show to ${ip}"
+		if [ -n "$out" ] ; then
 			echo "GOOD: node $pnn appears to have $ip assigned"
 		else
 			die "BAD: node $pnn does not appear to have $ip assigned"

ctdb/tests/simple/15_ctdb_statisticsreset.sh

@@ -37,10 +37,11 @@ num_nodes="$out"
 
 get_stat ()
 {
-    local label="$1"
-    local out="$2"
+	local label="$1"
 
-    echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
+	cat "$outfile" |
+	sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" |
+	head -1
 }
 
 check_reduced ()
@@ -59,20 +60,20 @@ check_reduced ()
 n=0
 while [ $n -lt $num_nodes ] ; do
     echo "Getting initial statistics for node ${n}..."
-    
+
     try_command_on_node -v $n $CTDB statistics
 
-    before_req_control=$(get_stat "req_control" "$out")
-    before_reply_control=$(get_stat "reply_control" "$out")
-    before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    before_req_control=$(get_stat "req_control")
+    before_reply_control=$(get_stat "reply_control")
+    before_node_packets_recv=$(get_stat "node_packets_recv")
 
     try_command_on_node $n $CTDB statisticsreset
 
     try_command_on_node -v $n $CTDB statistics
 
-    after_req_control=$(get_stat "req_control" "$out")
-    after_reply_control=$(get_stat "reply_control" "$out")
-    after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    after_req_control=$(get_stat "req_control")
+    after_reply_control=$(get_stat "reply_control")
+    after_node_packets_recv=$(get_stat "node_packets_recv")
 
     check_reduced "req_control" "$before_req_control" "$after_req_control"
     check_reduced "reply_control" "$before_reply_control" "$after_reply_control"

ctdb/tests/simple/19_ip_takeover_noop.sh

@@ -34,9 +34,7 @@ while read ip pnn ; do
 	if [ "$pnn" != "-1" ] ; then
 		die "BAD: IP address ${ip} is assigned to node ${pnn}"
 	fi
-done <<EOF
-$out
-EOF
+done <"$outfile"
 
 echo "GOOD: All IP addresses are unassigned"
 

ctdb/tests/simple/20_delip_iface_gc.sh

@@ -19,14 +19,14 @@ select_test_node_and_ips
 
 # Find interfaces on test node
 try_command_on_node $test_node "$CTDB ifaces -X"
-ifaces=$(awk -F'|' 'NR > 1 { print $2 }' <<<"$out")
+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
 echo "Node ${test_node} has interfaces: ${ifaces}"
 
 # Delete all IPs on each interface...  deleting IPs from one interface
 # can cause other interfaces to disappear, so we need to be careful...
 for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
 
     if [ -z "$info" ] ; then
 	echo "Interface ${i} missing... assuming already deleted!"
@@ -38,7 +38,7 @@ for i in $ifaces ; do
 
     try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
     awk -F'|' -v i="$i" \
-	'$6 == i { print $2 }' <<<"$out" |
+	'$6 == i { print $2 }' "$outfile" |
     while read ip ; do
 	echo "  $ip"
 	try_command_on_node $test_node "$CTDB delip $ip"
@@ -46,8 +46,8 @@ for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ipreallocate"
 
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
-    
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
     if [ -z "$info" ] ; then
 	echo "GOOD: Interface ${i} has been garbage collected"
     else

ctdb/tests/simple/21_ctdb_attach.sh

@@ -36,9 +36,8 @@ cluster_is_healthy
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 lastnode=$(( numnodes - 1 ))
 
 ######################################################################
@@ -55,7 +54,7 @@ check_db ()
 	echo "$out"
 	exit 1
     else
-	local flags=$(awk '{print $4}' <<<"$out") || true
+	local flags=$(awk '{print $4}' "$outfile") || true
 	if [ "$flags" = "$flag" ]; then
 	    echo "GOOD: database $db is attached on node $node with flag $flag"
 	else

ctdb/tests/simple/23_ctdb_moveip.sh

@@ -23,7 +23,28 @@ cluster_is_healthy
 
 select_test_node_and_ips
 
-sanity_check_ips "$out"
+sanity_check_ips ()
+{
+    echo "Sanity checking IPs..."
+
+    local x ipp prev
+    prev=""
+    while read x ipp ; do
+	[ "$ipp" = "-1" ] && break
+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+	    echo "OK"
+	    return 0
+	fi
+	prev="$ipp"
+    done <"$outfile"
+
+    echo "BAD: a node was -1 or IPs are only assigned to one node:"
+    cat "$outfile"
+    echo "Are you running an old version of CTDB?"
+    return 1
+}
+
+sanity_check_ips
 
 # Find a target node - it must be willing to host $test_ip
 try_command_on_node any "$CTDB listnodes | wc -l"
@@ -37,7 +58,7 @@ for i in $(seq 0 $(($num_nodes - 1)) ) ; do
 	    to_node="$i"
 	    break 2
 	fi
-    done <<<"$out"
+    done <"$outfile"
 done
 
 if [ -z "$to_node" ] ; then

ctdb/tests/simple/24_ctdb_getdbmap.sh

@@ -44,7 +44,7 @@ db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[
 
 sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
 
-num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+num_db_init=$(sed -n -e '1s/.*://p' "$outfile")
 
 for i in $(seq 1 5) ; do
     f=$(make_temp_db_filename)
@@ -52,14 +52,14 @@ for i in $(seq 1 5) ; do
     try_command_on_node 0 $CTDB attach "$f"
     try_command_on_node 0 $CTDB getdbmap
     sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
-    num=$(echo "$out" | sed -n -e '1s/^.*://p')
+    num=$(sed -n -e '1s/^.*://p' "$outfile")
     if [ $num = $(($num_db_init + $i)) ] ; then
 	echo "OK: correct number of additional databases"
     else
 	echo "BAD: no additional database"
 	exit 1
     fi
-    if [ "${out/name:${f} /}" != "$out" ] ; then
+    if awk '{print $2}' "$outfile" | grep -Fqx "name:$f" ; then
 	echo "OK: getdbmap knows about \"$f\""
     else
 	echo "BAD: getdbmap does not know about \"$f\""

ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh

@@ -42,8 +42,8 @@ cluster_is_healthy
 
 test_node=1
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 echo "There are $num_nodes nodes."
 
 echo "Shutting down node ${test_node}..."
@@ -58,7 +58,7 @@ pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error:
 for i in ip disable enable "ban 0" unban listvars ; do
     try_command_on_node -v 0 ! $CTDB $i -n $test_node
 
-    if egrep -q "$pat" <<<"$out" ; then
+    if egrep -q "$pat" "$outfile" ; then
 	echo "OK: \"ctdb ${i}\" fails with expected \"disconnected node\" message"
     else
 	echo "BAD: \"ctdb ${i}\" does not fail with expected \"disconnected node\" message"

ctdb/tests/simple/27_ctdb_detach.sh

@@ -32,9 +32,8 @@ cluster_is_healthy
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 
 ######################################################################
 
@@ -43,12 +42,12 @@ check_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq $numnodes ]; then
 	echo "GOOD: database $db is attached on all nodes"
     else
 	echo "BAD: database $db is not attached on all nodes"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -58,12 +57,12 @@ check_no_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq 0 ]; then
 	echo "GOOD: database $db is not attached any more"
     else
 	echo "BAD: database $db is still attached"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -134,12 +133,12 @@ echo
 echo "Write a key to database"
 try_command_on_node 0 $CTDB writekey $testdb1 foo bar
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 1 ]; then
     echo "GOOD: Key added to database"
 else
     echo "BAD: Key did not get added to database"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi
 
@@ -158,11 +157,11 @@ check_db "$testdb1"
 echo
 echo "Check if the database is empty"
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 0 ]; then
     echo "GOOD: Database $testdb1 is empty"
 else
     echo "BAD: Database $testdb1 is not empty"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi

ctdb/tests/simple/35_ctdb_getreclock.sh

@@ -25,7 +25,7 @@ if [ -z "$out" ] ; then
     exit 0
 fi
 
-n=$(echo "$out" | sort -u | wc -l)
+n=$(sort -u "$outfile" | wc -l)
 if [ "$n" = 1 ] ; then
 	echo "GOOD: All nodes have the same recovery lock setting"
 else

ctdb/tests/simple/51_message_ring.sh

@@ -19,23 +19,21 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running message_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND message_ring -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\))$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
-stuff="${prev##Ring\[*\]: }"
+stuff="${last##Ring\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then

ctdb/tests/simple/52_fetch_ring.sh

@@ -19,8 +19,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running fetch_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
@@ -29,13 +29,11 @@ pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)?
 sanity_check_output 1 "$pat"
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
-# $prev should look like this:
+# $last should look like this:
 #    Fetch[1]: 10670.93 msgs/sec
-stuff="${prev##*Fetch\[*\]: }"
+stuff="${last##*Fetch\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then

ctdb/tests/simple/53_transaction_loop.sh

@@ -24,8 +24,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30

ctdb/tests/simple/54_transaction_loop_recovery.sh

@@ -43,8 +43,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30

ctdb/tests/simple/55_ctdb_ptrans.sh

@@ -51,11 +51,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records were inserted"
@@ -75,11 +75,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records found"
@@ -116,11 +116,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 0 ] ; then
     echo "BAD: expected 0 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 0 records found"

ctdb/tests/simple/56_replicated_transaction_recovery.sh

@@ -44,8 +44,8 @@ TESTDB="replicated_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB replicated"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30

ctdb/tests/simple/58_ctdb_restoredb.sh

@@ -36,10 +36,10 @@ set -e
 cluster_is_healthy
 
 try_command_on_node 0 $CTDB status
-generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="restoredb_test.tdb"
@@ -105,7 +105,7 @@ fi
 wait_until_ready
 
 try_command_on_node 0 $CTDB status
-new_generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+new_generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
 echo "Old generation = $generation"
 echo "New generation = $new_generation"

ctdb/tests/simple/69_recovery_resurrect_deleted.sh

@@ -20,9 +20,9 @@ testdb="rec_test.tdb"
 echo "Getting list of nodes..."
 try_command_on_node -v any "onnode -pq all ctdb pnn | grep '^[0-9][0-9]*$'"
 
-first=$(echo "$out" | sed -n -e '1p')
-second=$(echo "$out" | sed -n -e '2p')
-notfirst=$(echo "$out" | tail -n +2)
+first=$(sed -n -e '1p' "$outfile")
+second=$(sed -n -e '2p' "$outfile")
+notfirst=$(tail -n +2 "$outfile")
 
 echo "Create/wipe test database ${testdb}"
 try_command_on_node $first $CTDB attach "$testdb"
@@ -46,7 +46,7 @@ database_has_zero_records ()
 	local n
 	for n in $notfirst ; do
 		try_command_on_node $n $CTDB cattdb "$testdb"
-		if echo "$out" | grep -q '^key(' ; then
+		if grep -q '^key(' "$outfile" ; then
 			return 1
 		fi
 	done
@@ -74,7 +74,7 @@ wait_until_node_has_status $first notstopped
 echo "Get database contents"
 try_command_on_node -v $first $CTDB catdb "$testdb"
 
-if echo "$out" | grep -q '^key(' ; then
+if grep -q '^key(' "$outfile" ; then
 	echo "BAD: Deleted record has been resurrected"
 	exit 1
 fi

ctdb/tests/simple/70_recoverpdbbyseqnum.sh

@@ -35,8 +35,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 add_record_per_node ()
 {

ctdb/tests/simple/71_ctdb_wipedb.sh

@@ -32,8 +32,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="persistent_test.tdb"

ctdb/tests/simple/72_update_record_persistent.sh

@@ -34,8 +34,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 test_db="persistent_test.tdb"
 

ctdb/tests/simple/75_readonly_records_basic.sh

@@ -49,12 +49,12 @@ check_no_readonly ()
 {
     try_command_on_node all $CTDB cattdb $testdb
     local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS"
-    local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true
+    local numreadonly=$(grep -c -E "$ro_flags" "$outfile") || true
     if [ $numreadonly -eq 0 ] ; then
 	echo "GOOD: no read-only delegations"
     else
 	echo "BAD: there are read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -72,36 +72,36 @@ check_readonly ()
     local count
 
     try_command_on_node $dmaster $CTDB cattdb $testdb
-    count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
     if [ $count -eq 1 ] ; then
 	echo "GOOD: dmaster ${dmaster} has read-only delegations"
     else
 	echo "BAD: dmaster ${dmaster} has no read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
-    count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
     if [ $count -ne 0 ] ; then
 	echo "BAD: dmaster ${dmaster} has a read-only copy"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 
     local o
     for o in $others ; do
 	try_command_on_node $o $CTDB cattdb $testdb
-	count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
 	if [ $count -eq 1 ] ; then
 	    echo "GOOD: node ${o} has a read-only copy"
 	else
 	    echo "BAD: node ${o} has no read-only copy"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
-	count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
 	if [ $count -ne 0 ] ; then
 	    echo "BAD: other node ${o} has read-only delegations"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
     done
@@ -111,7 +111,7 @@ check_readonly ()
 
 echo "Get list of nodes..."
 try_command_on_node any $CTDB -X listnodes
-all_nodes=$(awk -F'|' '{print $2}' <<<"$out")
+all_nodes=$(awk -F'|' '{print $2}' "$outfile")
 
 ######################################################################
 
@@ -138,7 +138,7 @@ try_command_on_node all $CTDB setdbreadonly $testdb
 
 # Database should be tagged as READONLY
 try_command_on_node 0 $CTDB getdbmap
-db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out")
+db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' "$outfile")
 if grep -q "READONLY" <<<"$db_details" ; then
     echo "GOOD: read-only record support is enabled"
 else

ctdb/tests/simple/77_ctdb_db_recovery.sh

@@ -93,7 +93,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value1
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -108,7 +108,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value2
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -119,7 +119,7 @@ wait_until_node_has_status $recmaster recovered
 # Verify record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 if [ "$out" = "Data: size:6 ptr:[value2]" ]; then
 	echo "GOOD: Recovery did not corrupt database"
 else

ctdb/tests/simple/79_volatile_db_traverse.sh

@@ -75,7 +75,7 @@ try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1"
 echo "do traverse on node 0"
 try_command_on_node -v 0 $CTDB catdb "$TESTDB"
 
-num=$(echo "$out" | sed -n -e 's|^Dumped \(.*\) records$|\1|p')
+num=$(sed -n -e 's|^Dumped \(.*\) records$|\1|p' "$outfile")
 if [ "$num" = 1 ] ; then
 	echo "OK: There was 1 record"
 else
@@ -83,7 +83,7 @@ else
 	exit 1
 fi
 
-if echo "$out" | grep -q "^data(4) = \"bar1\"\$" ; then
+if grep -q "^data(4) = \"bar1\"\$" "$outfile" ; then
 	echo "OK: Data from node 1 was returned"
 else
 	echo "BAD: Data from node 1 was not returned"

ctdb/tests/simple/80_ctdb_traverse.sh

@@ -58,7 +58,7 @@ done
 echo "Start a traverse and collect records"
 try_command_on_node 0 $CTDB catdb $TESTDB
 
-num_read=$(echo "$out" | tail -n 1 | cut -d\  -f2)
+num_read=$(tail -n 1 "$outfile" | cut -d\  -f2)
 if [ $num_read -eq $num_records ]; then
 	echo "GOOD: All $num_records records retrieved"
 	status=0

ctdb/tests/simple/81_tunnel_ring.sh

@@ -19,24 +19,22 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running tunnel_test on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND \
 	tunnel_test -t 30 -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    pnn[2] count=85400
-stuff="${prev##pnn\[*\] }"
+stuff="${last##pnn\[*\] }"
 mps="${stuff% msgs/sec}"
 
 if [ ${mps%.*} -ge 10 ] ; then

ctdb/tests/simple/90_debug_hung_script.sh

@@ -54,11 +54,9 @@ wait_for_monitor_event $test_node
 
 echo "Waiting for debugging output to appear..."
 # Use test -s because the file is created above using mktemp
-wait_until 60 onnode $test_node test -s "$debug_output"
+wait_until 60 test -s "$debug_output"
 
 echo "Checking output of hung script debugging..."
-try_command_on_node -v $test_node cat "$debug_output"
-hung_script_output="$out"
 
 # Can we actually read kernel stacks
 if try_command_on_node $test_node "cat /proc/$$/stack >/dev/null 2>&1" ; then
@@ -72,7 +70,7 @@ fi
 
 while IFS="" read pattern ; do
     [ -n "$pattern" ] || continue
-    if grep -- "^${pattern}\$" <<<"$hung_script_output" >/dev/null ; then
+    if grep -q -- "^${pattern}\$" "$debug_output" ; then
 	printf 'GOOD: output contains "%s"\n' "$pattern"
     else
 	printf 'BAD: output does not contain "%s"\n' "$pattern"