From 89b49205391105c6ce2b058531c79d939fe97007 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 7 Dec 2016 09:42:46 +1100
Subject: [PATCH] ctdb-tests: Add tests for takeover helper

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
---
 ctdb/tests/run_tests.sh                     |   3 +-
 ctdb/tests/takeover_helper/000.sh           |  22 ++++
 ctdb/tests/takeover_helper/010.sh           |  33 ++++++
 ctdb/tests/takeover_helper/011.sh           |  33 ++++++
 ctdb/tests/takeover_helper/012.sh           |  33 ++++++
 ctdb/tests/takeover_helper/013.sh           |  33 ++++++
 ctdb/tests/takeover_helper/014.sh           |  33 ++++++
 ctdb/tests/takeover_helper/015.sh           |  39 +++++++
 ctdb/tests/takeover_helper/016.sh           |  36 +++++++
 ctdb/tests/takeover_helper/017.sh           |  36 +++++++
 ctdb/tests/takeover_helper/018.sh           |  34 ++++++
 ctdb/tests/takeover_helper/019.sh           |  37 +++++++
 ctdb/tests/takeover_helper/020.sh           |  36 +++++++
 ctdb/tests/takeover_helper/021.sh           |  39 +++++++
 ctdb/tests/takeover_helper/022.sh           |  40 +++++++
 ctdb/tests/takeover_helper/023.sh           |  41 ++++++++
 ctdb/tests/takeover_helper/024.sh           |  43 ++++++++
 ctdb/tests/takeover_helper/025.sh           |  37 +++++++
 ctdb/tests/takeover_helper/026.sh           |  41 ++++++++
 ctdb/tests/takeover_helper/110.sh           |  29 ++++++
 ctdb/tests/takeover_helper/111.sh           |  40 +++++++
 ctdb/tests/takeover_helper/120.sh           |  40 +++++++
 ctdb/tests/takeover_helper/121.sh           |  40 +++++++
 ctdb/tests/takeover_helper/122.sh           |  40 +++++++
 ctdb/tests/takeover_helper/130.sh           |  41 ++++++++
 ctdb/tests/takeover_helper/131.sh           |  40 +++++++
 ctdb/tests/takeover_helper/132.sh           |  42 ++++++++
 ctdb/tests/takeover_helper/140.sh           |  32 ++++++
 ctdb/tests/takeover_helper/150.sh           |  31 ++++++
 ctdb/tests/takeover_helper/160.sh           |  31 ++++++
 ctdb/tests/takeover_helper/210.sh           |  29 ++++++
 ctdb/tests/takeover_helper/211.sh           |  40 +++++++
 ctdb/tests/takeover_helper/220.sh           |  40 +++++++
 ctdb/tests/takeover_helper/230.sh           |  41 ++++++++
 ctdb/tests/takeover_helper/240.sh           |  32 ++++++
 ctdb/tests/takeover_helper/250.sh           |  31 ++++++
 ctdb/tests/takeover_helper/260.sh           |  31 ++++++
 ctdb/tests/takeover_helper/scripts/local.sh | 109 ++++++++++++++++++++
 ctdb/wscript                                |   1 +
 39 files changed, 1408 insertions(+), 1 deletion(-)
 create mode 100755 ctdb/tests/takeover_helper/000.sh
 create mode 100755 ctdb/tests/takeover_helper/010.sh
 create mode 100755 ctdb/tests/takeover_helper/011.sh
 create mode 100755 ctdb/tests/takeover_helper/012.sh
 create mode 100755 ctdb/tests/takeover_helper/013.sh
 create mode 100755 ctdb/tests/takeover_helper/014.sh
 create mode 100755 ctdb/tests/takeover_helper/015.sh
 create mode 100755 ctdb/tests/takeover_helper/016.sh
 create mode 100755 ctdb/tests/takeover_helper/017.sh
 create mode 100755 ctdb/tests/takeover_helper/018.sh
 create mode 100755 ctdb/tests/takeover_helper/019.sh
 create mode 100755 ctdb/tests/takeover_helper/020.sh
 create mode 100755 ctdb/tests/takeover_helper/021.sh
 create mode 100755 ctdb/tests/takeover_helper/022.sh
 create mode 100755 ctdb/tests/takeover_helper/023.sh
 create mode 100755 ctdb/tests/takeover_helper/024.sh
 create mode 100755 ctdb/tests/takeover_helper/025.sh
 create mode 100755 ctdb/tests/takeover_helper/026.sh
 create mode 100755 ctdb/tests/takeover_helper/110.sh
 create mode 100755 ctdb/tests/takeover_helper/111.sh
 create mode 100755 ctdb/tests/takeover_helper/120.sh
 create mode 100755 ctdb/tests/takeover_helper/121.sh
 create mode 100755 ctdb/tests/takeover_helper/122.sh
 create mode 100755 ctdb/tests/takeover_helper/130.sh
 create mode 100755 ctdb/tests/takeover_helper/131.sh
 create mode 100755 ctdb/tests/takeover_helper/132.sh
 create mode 100755 ctdb/tests/takeover_helper/140.sh
 create mode 100755 ctdb/tests/takeover_helper/150.sh
 create mode 100755 ctdb/tests/takeover_helper/160.sh
 create mode 100755 ctdb/tests/takeover_helper/210.sh
 create mode 100755 ctdb/tests/takeover_helper/211.sh
 create mode 100755 ctdb/tests/takeover_helper/220.sh
 create mode 100755 ctdb/tests/takeover_helper/230.sh
 create mode 100755 ctdb/tests/takeover_helper/240.sh
 create mode 100755 ctdb/tests/takeover_helper/250.sh
 create mode 100755 ctdb/tests/takeover_helper/260.sh
 create mode 100644 ctdb/tests/takeover_helper/scripts/local.sh

diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh
index c39b92dcd85..f128d959252 100755
--- a/ctdb/tests/run_tests.sh
+++ b/ctdb/tests/run_tests.sh
@@ -257,7 +257,8 @@ export TEST_SCRIPTS_DIR="${test_dir}/scripts"
 # If no tests specified then run some defaults
 if [ -z "$1" ] ; then
     if [ -n "$TEST_LOCAL_DAEMONS" ] ; then
-	set -- onnode takeover tool eventscripts cunit eventd shellcheck simple
+	set -- onnode takeover takeover_helper tool eventscripts \
+	    cunit eventd shellcheck simple
     else
 	set -- simple complex
     fi
diff --git a/ctdb/tests/takeover_helper/000.sh b/ctdb/tests/takeover_helper/000.sh
new file mode 100755
index 00000000000..3cb9635ad3a
--- /dev/null
+++ b/ctdb/tests/takeover_helper/000.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+EOF
+
+required_result 0 <<EOF
+No nodes available to host public IPs yet
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/010.sh b/ctdb/tests/takeover_helper/010.sh
new file mode 100755
index 00000000000..12751566201
--- /dev/null
+++ b/ctdb/tests/takeover_helper/010.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/011.sh b/ctdb/tests/takeover_helper/011.sh
new file mode 100755
index 00000000000..12a2a1a22e3
--- /dev/null
+++ b/ctdb/tests/takeover_helper/011.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 ok, IPs all unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 2
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/012.sh b/ctdb/tests/takeover_helper/012.sh
new file mode 100755
index 00000000000..04e4508a2be
--- /dev/null
+++ b/ctdb/tests/takeover_helper/012.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 1 IP unassigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 2
+10.0.0.33 1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/013.sh b/ctdb/tests/takeover_helper/013.sh
new file mode 100755
index 00000000000..ad55564b4ed
--- /dev/null
+++ b/ctdb/tests/takeover_helper/013.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 unhealthy, IPs all assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 0
+10.0.0.33 1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/014.sh b/ctdb/tests/takeover_helper/014.sh
new file mode 100755
index 00000000000..bb368939142
--- /dev/null
+++ b/ctdb/tests/takeover_helper/014.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, all IPs assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ok_null
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 2
+10.0.0.33 1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/015.sh b/ctdb/tests/takeover_helper/015.sh
new file mode 100755
index 00000000000..2841412b68c
--- /dev/null
+++ b/ctdb/tests/takeover_helper/015.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, all IPs assigned, NoIPHostOnAllDisabled"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  2
+10.0.0.33  1
+EOF
+
+ctdb_cmd setvar NoIPHostOnAllDisabled 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.33
+Failed to find node to cover ip 10.0.0.32
+Failed to find node to cover ip 10.0.0.31
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/016.sh b/ctdb/tests/takeover_helper/016.sh
new file mode 100755
index 00000000000..4d2e4fea209
--- /dev/null
+++ b/ctdb/tests/takeover_helper/016.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs all unassigned, DisableIPFailover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+EOF
+
+ctdb_cmd setvar DisableIPFailover 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/017.sh b/ctdb/tests/takeover_helper/017.sh
new file mode 100755
index 00000000000..e5bcd209fad
--- /dev/null
+++ b/ctdb/tests/takeover_helper/017.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs unbalanced, NoIPFailback"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  1
+EOF
+
+ctdb_cmd setvar NoIPFailback 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/018.sh b/ctdb/tests/takeover_helper/018.sh
new file mode 100755
index 00000000000..61a26dd0540
--- /dev/null
+++ b/ctdb/tests/takeover_helper/018.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs unbalanced"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  1
+EOF
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/019.sh b/ctdb/tests/takeover_helper/019.sh
new file mode 100755
index 00000000000..0802611bab0
--- /dev/null
+++ b/ctdb/tests/takeover_helper/019.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, 1 node unhealthy, IPs all assigned, NoIPTakeover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.32
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 -1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/020.sh b/ctdb/tests/takeover_helper/020.sh
new file mode 100755
index 00000000000..111b5a4bfa1
--- /dev/null
+++ b/ctdb/tests/takeover_helper/020.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all unhealthy, IPs all assigned, NoIPTakeover"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x2     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x2
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/021.sh b/ctdb/tests/takeover_helper/021.sh
new file mode 100755
index 00000000000..ad8e59f3dea
--- /dev/null
+++ b/ctdb/tests/takeover_helper/021.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs all assigned"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  -1
+EOF
+
+ctdb_cmd setvar NoIPTakeover 1
+
+ok <<EOF
+Failed to find node to cover ip 10.0.0.34
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/022.sh b/ctdb/tests/takeover_helper/022.sh
new file mode 100755
index 00000000000..e8c5a961c7d
--- /dev/null
+++ b/ctdb/tests/takeover_helper/022.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, no force rebalance"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 2
+10.0.0.36 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/023.sh b/ctdb/tests/takeover_helper/023.sh
new file mode 100755
index 00000000000..a76afef7c8b
--- /dev/null
+++ b/ctdb/tests/takeover_helper/023.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, force rebalance 1"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+Forcing rebalancing of IPs to node 1
+EOF
+test_takeover_helper 1
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 1
+10.0.0.36 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/024.sh b/ctdb/tests/takeover_helper/024.sh
new file mode 100755
index 00000000000..af7480cb3d8
--- /dev/null
+++ b/ctdb/tests/takeover_helper/024.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all healthy, IPs very unbalanced, force rebalance all"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+10.0.0.36  2
+EOF
+
+ok <<EOF
+Forcing rebalancing of IPs to node 1
+Forcing rebalancing of IPs to node 0
+Forcing rebalancing of IPs to node 2
+EOF
+test_takeover_helper 1,0,2
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 0
+10.0.0.36 1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/025.sh b/ctdb/tests/takeover_helper/025.sh
new file mode 100755
index 00000000000..28db4868391
--- /dev/null
+++ b/ctdb/tests/takeover_helper/025.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all assigned randomly, deterministic IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  0
+10.0.0.33  2
+EOF
+
+ctdb_cmd setvar IPAllocAlgorithm 0
+
+ok <<EOF
+Deterministic IPs enabled. Resetting all ip allocations
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/026.sh b/ctdb/tests/takeover_helper/026.sh
new file mode 100755
index 00000000000..08a7b6d8f21
--- /dev/null
+++ b/ctdb/tests/takeover_helper/026.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs assigned, unbalanced, non-deterministic IPs"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+10.0.0.34  2
+10.0.0.35  2
+EOF
+
+ctdb_cmd setvar IPAllocAlgorithm 1
+
+ok_null
+test_takeover_helper
+
+# This is non-deterministic - LCP2 would not rebalance without
+# force-rebalance-nodes
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+10.0.0.34 2
+10.0.0.35 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/110.sh b/ctdb/tests/takeover_helper/110.sh
new file mode 100755
index 00000000000..56dc16c285f
--- /dev/null
+++ b/ctdb/tests/takeover_helper/110.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs, IPREALLOCATED error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+CONTROLFAILS
+137	1	ERROR	CTDB_CONTROL_IPREALLOCATED fake failure
+
+EOF
+
+required_result 255 <<EOF
+No nodes available to host public IPs yet
+IPREALLOCATED failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/111.sh b/ctdb/tests/takeover_helper/111.sh
new file mode 100755
index 00000000000..d14868bd234
--- /dev/null
+++ b/ctdb/tests/takeover_helper/111.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, IPREALLOCATED error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+137	1	ERROR	CTDB_CONTROL_IPREALLOCATED fake failure
+EOF
+
+required_result 255 <<EOF
+IPREALLOCATED failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/120.sh b/ctdb/tests/takeover_helper/120.sh
new file mode 100755
index 00000000000..af780d6453d
--- /dev/null
+++ b/ctdb/tests/takeover_helper/120.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, TAKEOVER_IP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+89	1	ERROR	CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.32 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 -1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/121.sh b/ctdb/tests/takeover_helper/121.sh
new file mode 100755
index 00000000000..cc113dac2a0
--- /dev/null
+++ b/ctdb/tests/takeover_helper/121.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 2/3 IPs assigned, TAKEOVER_IP error (redundant)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  2
+10.0.0.32  1
+10.0.0.33  -1
+
+CONTROLFAILS
+89	1	ERROR	CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.32 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/122.sh b/ctdb/tests/takeover_helper/122.sh
new file mode 100755
index 00000000000..d823b097900
--- /dev/null
+++ b/ctdb/tests/takeover_helper/122.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, 2/3 IPs assigned, TAKEOVER_IP error (target)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  2
+10.0.0.32  1
+10.0.0.33  -1
+
+CONTROLFAILS
+89	0	ERROR	CTDB_CONTROL_TAKEOVER_IP fake failure
+EOF
+
+required_result 255 <<EOF
+TAKEOVER_IP 10.0.0.33 failed on node 0, ret=-1
+Assigning banning credits to node 0
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/130.sh b/ctdb/tests/takeover_helper/130.sh
new file mode 100755
index 00000000000..83735d4019d
--- /dev/null
+++ b/ctdb/tests/takeover_helper/130.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, RELEASE_IP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+88	2	ERROR	CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 2, ret=-1
+RELEASE_IP 10.0.0.32 failed on node 2, ret=-1
+Assigning banning credits to node 2
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/131.sh b/ctdb/tests/takeover_helper/131.sh
new file mode 100755
index 00000000000..4e0cd46c354
--- /dev/null
+++ b/ctdb/tests/takeover_helper/131.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, all IPs assigned, RELEASE_IP error (redundant)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+
+CONTROLFAILS
+88	0	ERROR	CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 0, ret=-1
+Assigning banning credits to node 0
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 -1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/132.sh b/ctdb/tests/takeover_helper/132.sh
new file mode 100755
index 00000000000..a1a4ce53cfa
--- /dev/null
+++ b/ctdb/tests/takeover_helper/132.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, all IPs assigned, RELEASE_IP error (target)"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x2
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  0
+10.0.0.32  1
+10.0.0.33  2
+
+CONTROLFAILS
+88	1	ERROR	CTDB_CONTROL_RELEASE_IP fake failure
+EOF
+
+required_result 255 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 1, ret=-1
+RELEASE_IP 10.0.0.32 failed on node 1, ret=-1
+RELEASE_IP 10.0.0.31 failed on node 1, ret=-1
+Assigning banning credits to node 1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 0
+10.0.0.32 1
+10.0.0.33 2
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/140.sh b/ctdb/tests/takeover_helper/140.sh
new file mode 100755
index 00000000000..60a03f229b4
--- /dev/null
+++ b/ctdb/tests/takeover_helper/140.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_PUBLIC_IPS error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+90	2	ERROR	CTDB_CONTROL_GET_PUBLIC_IPS fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_PUBLIC_IPS failed on node 2, ret=-1
+Failed to fetch known public IPs
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/150.sh b/ctdb/tests/takeover_helper/150.sh
new file mode 100755
index 00000000000..56042b4b1be
--- /dev/null
+++ b/ctdb/tests/takeover_helper/150.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_NODEMAP error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+91	0	ERROR	CTDB_CONTROL_GET_NODEMAP fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_NODEMAP failed, ret=-1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/160.sh b/ctdb/tests/takeover_helper/160.sh
new file mode 100755
index 00000000000..c09f6495d06
--- /dev/null
+++ b/ctdb/tests/takeover_helper/160.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_ALL_TUNABLES error"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+53	0	ERROR	CTDB_CONTROL_GET_ALL_TUNABLES fake failure
+EOF
+
+required_result 255 <<EOF
+control GET_ALL_TUNABLES failed, ret=-1
+takeover run failed, ret=-1
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/210.sh b/ctdb/tests/takeover_helper/210.sh
new file mode 100755
index 00000000000..207949e1165
--- /dev/null
+++ b/ctdb/tests/takeover_helper/210.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, no IPs, IPREALLOCATED timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+CONTROLFAILS
+137	1	TIMEOUT	CTDB_CONTROL_IPREALLOCATED fake timeout
+
+EOF
+
+required_result 110 <<EOF
+No nodes available to host public IPs yet
+IPREALLOCATED failed on node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/211.sh b/ctdb/tests/takeover_helper/211.sh
new file mode 100755
index 00000000000..f4eca683d08
--- /dev/null
+++ b/ctdb/tests/takeover_helper/211.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, IPREALLOCATED timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+137	1	TIMEOUT	CTDB_CONTROL_IPREALLOCATED fake timeout
+EOF
+
+required_result 110 <<EOF
+IPREALLOCATED failed on node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/220.sh b/ctdb/tests/takeover_helper/220.sh
new file mode 100755
index 00000000000..d8c7823c038
--- /dev/null
+++ b/ctdb/tests/takeover_helper/220.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, TAKEOVER_IP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+89	1	TIMEOUT	CTDB_CONTROL_TAKEOVER_IP fake timeout
+EOF
+
+required_result 110 <<EOF
+TAKEOVER_IP 10.0.0.32 failed to node 1, ret=110
+Assigning banning credits to node 1
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 2
+10.0.0.32 -1
+10.0.0.33 0
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/230.sh b/ctdb/tests/takeover_helper/230.sh
new file mode 100755
index 00000000000..9b77c2b6f76
--- /dev/null
+++ b/ctdb/tests/takeover_helper/230.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, IPs all unassigned, RELEASE_IP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  -1
+10.0.0.32  -1
+10.0.0.33  -1
+
+CONTROLFAILS
+88	2	TIMEOUT	CTDB_CONTROL_RELEASE_IP fake timeout
+EOF
+
+required_result 110 <<EOF
+RELEASE_IP 10.0.0.33 failed on node 2, ret=110
+RELEASE_IP 10.0.0.32 failed on node 2, ret=110
+Assigning banning credits to node 2
+takeover run failed, ret=110
+EOF
+test_takeover_helper
+
+required_result 0 <<EOF
+Public IPs on ALL nodes
+10.0.0.31 -1
+10.0.0.32 -1
+10.0.0.33 -1
+EOF
+test_ctdb_ip_all
diff --git a/ctdb/tests/takeover_helper/240.sh b/ctdb/tests/takeover_helper/240.sh
new file mode 100755
index 00000000000..00004a5fffa
--- /dev/null
+++ b/ctdb/tests/takeover_helper/240.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_PUBLIC_IPS timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+90	2	TIMEOUT	CTDB_CONTROL_GET_PUBLIC_IPS fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_PUBLIC_IPS failed on node 2, ret=110
+Failed to fetch known public IPs
+takeover run failed, ret=110
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/250.sh b/ctdb/tests/takeover_helper/250.sh
new file mode 100755
index 00000000000..1422474883e
--- /dev/null
+++ b/ctdb/tests/takeover_helper/250.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_NODEMAP timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+91	0	TIMEOUT	CTDB_CONTROL_GET_NODEMAP fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_NODEMAP failed to node 0, ret=110
+takeover run failed, ret=110
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/260.sh b/ctdb/tests/takeover_helper/260.sh
new file mode 100755
index 00000000000..645faa2c6cb
--- /dev/null
+++ b/ctdb/tests/takeover_helper/260.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "3 nodes, all ok, GET_ALL_TUNABLES timeout"
+
+setup_ctdbd <<EOF
+NODEMAP
+0       192.168.20.41   0x0     CURRENT RECMASTER
+1       192.168.20.42   0x0
+2       192.168.20.43   0x0
+
+IFACES
+:Name:LinkStatus:References:
+:eth2:1:2:
+:eth1:1:4:
+
+PUBLICIPS
+10.0.0.31  1
+10.0.0.32  1
+10.0.0.33  1
+
+CONTROLFAILS
+53	0	TIMEOUT	CTDB_CONTROL_GET_ALL_TUNABLES fake timeout
+EOF
+
+required_result 110 <<EOF
+control GET_ALL_TUNABLES failed, ret=110
+takeover run failed, ret=110
+EOF
+test_takeover_helper
diff --git a/ctdb/tests/takeover_helper/scripts/local.sh b/ctdb/tests/takeover_helper/scripts/local.sh
new file mode 100644
index 00000000000..4dd20d7ebbc
--- /dev/null
+++ b/ctdb/tests/takeover_helper/scripts/local.sh
@@ -0,0 +1,109 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# Augment PATH with stubs/ directory.
+
+if "$TEST_VERBOSE" ; then
+	debug () { echo "$@" ; }
+else
+	debug () { : ; }
+fi
+
+. "${TEST_SCRIPTS_DIR}/script_install_paths.sh"
+
+PATH="$PATH:$CTDB_SCRIPTS_TOOLS_HELPER_DIR"
+
+ctdbd_socket="${TEST_VAR_DIR}/ctdbd.socket.$$"
+ctdbd_pidfile="${TEST_VAR_DIR}/ctdbd.pid.$$"
+
+define_test ()
+{
+	_f=$(basename "$0" ".sh")
+
+	printf "%-28s - %s\n" "$_f" "$1"
+
+	if [ -z "$FAKE_CTDBD_DEBUGLEVEL" ] ; then
+		FAKE_CTDBD_DEBUGLEVEL="ERR"
+	fi
+	if [ -z "$HELPER_DEBUGLEVEL" ] ; then
+		HELPER_DEBUGLEVEL="NOTICE"
+	fi
+	if [ -z "$CTDB_DEBUGLEVEL" ] ; then
+		CTDB_DEBUGLEVEL="ERR"
+	fi
+}
+
+cleanup_ctdbd ()
+{
+	debug "Cleaning up fake ctdbd"
+
+	pid=$(cat "$ctdbd_pidfile" 2>/dev/null || echo)
+	if [ -n "$pid" ] ; then
+		kill $pid || true
+		rm -f "$ctdbd_pidfile"
+	fi
+	rm -f "$ctdbd_socket"
+}
+
+setup_ctdbd ()
+{
+	debug "Setting up fake ctdbd"
+
+	$VALGRIND fake_ctdbd -d "$FAKE_CTDBD_DEBUGLEVEL" \
+		  -s "$ctdbd_socket" -p "$ctdbd_pidfile"
+	# This current translates to a 6 second timeout for the
+	# important controls
+	ctdb --socket $ctdbd_socket setvar TakeoverTimeout 2
+	test_cleanup cleanup_ctdbd
+}
+
+ctdbd_getpid ()
+{
+	cat "$ctdbd_pidfile"
+}
+
+# Render non-printable characters.  The helper prints the status as
+# binary, so render it for easy comparison.
+result_filter ()
+{
+	sed -e 's|ctdb-takeover\[[0-9]*\]: ||'
+}
+
+ctdb_cmd ()
+{
+	echo Running: ctdb -d "$CTDB_DEBUGLEVEL" --socket $ctdbd_socket "$@"
+	ctdb -d "$CTDB_DEBUGLEVEL" --socket $ctdbd_socket "$@"
+}
+
+test_ctdb_ip_all ()
+{
+	unit_test ctdb -d "$CTDB_DEBUGLEVEL" \
+		  --socket $ctdbd_socket ip all || exit $?
+}
+
+takeover_helper_out="${TEST_VAR_DIR}/takover_helper.out"
+
+takeover_helper_format_outfd ()
+{
+	od -A n -t d4 "$takeover_helper_out" | sed -e 's|^[[:space:]]*||'
+}
+
+test_takeover_helper ()
+{
+	(
+		export CTDB_DEBUGLEVEL="$HELPER_DEBUGLEVEL"
+		export CTDB_LOGGING="file:"
+		unit_test ctdb_takeover_helper 3 "$ctdbd_socket" "$@" \
+			  3>"$takeover_helper_out"
+	) || exit $?
+
+	case "$required_rc" in
+	255) _t="-1" ;;
+	*) _t="$required_rc" ;;
+	esac
+	ok "$_t"
+
+	unit_test_notrace takeover_helper_format_outfd
+	_ret=$?
+	rm "$takeover_helper_out"
+	[ $? -eq 0 ] || exit $?
+}
diff --git a/ctdb/wscript b/ctdb/wscript
index 66959cde091..b951dd69f6c 100644
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -838,6 +838,7 @@ def build(bld):
         'shellcheck',
         'simple',
         'takeover',
+        'takeover_helper',
         'tool'
     ]