andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

glusterd: allow multiple instances of glusterd on one machine

Browse Source 
This is needed to support automated testing of cluster-communication
features such as probing and quorum.  In order to use this, you need to
do the following preparatory steps.

* Copy /var/lib/glusterd to another directory for each virtual host

* Ensure that each virtual host has a different UUID in its glusterd.info

Now you can start each copy of glusterd with the following xlator-options.

* management.transport.socket.bind-address=$ip_address

* management.working-directory=$unique_working_directory

You can use 127.x.y.z addresses for binding without needing to assign
them to interfaces explicitly.  Note that you must use addresses, not
names, because of some stuff in the socket code that's not worth fixing
just for this usage, but after that you can use names in /etc/hosts
instead.

At this point you can issue CLI commands to a specific glusterd using
the --remote-host option.  So far probe, volume create/start/stop,
mount, and basic I/O all seem to work as expected with multiple
instances.

Change-Id: I1beabb44cff8763d2774bc208b2ffcda27c1a550
BUG: 913555
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: http://review.gluster.org/4556
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
This commit is contained in:
Jeff Darcy 2013-02-20 14:11:36 -05:00 committed by Anand Avati
parent 673287ae4d
commit 1dbe9a05fe
4 changed files with 142 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
tests/bugs/bug-913555.t Executable file
View File

@ -0,0 +1,66 @@
#!/bin/bash
# Test that a volume becomes unwritable when the cluster loses quorum.
. $(dirname $0)/../include.rc
. $(dirname $0)/../volume.rc
function vglusterd {
wd=$1/wd-$2
cp -r /var/lib/glusterd $wd
rm -rf $wd/peers/* $wd/vols/*
echo -n "UUID=$(uuidgen)\noperating-version=1\n" > $wd/glusterd.info
opt1="management.transport.socket.bind-address=127.0.0.$2"
opt2="management.working-directory=$wd"
glusterd --xlator-option $opt1 --xlator-option $opt2
}
function check_fs {
df $1 &> /dev/null
echo $?
}
function check_peers {
$VCLI peer status | grep 'Peer in Cluster (Connected)' | wc -l
}
cleanup;
topwd=$(mktemp -d)
trap "rm -rf $topwd" EXIT
vglusterd $topwd 100
VCLI="$CLI --remote-host=127.0.0.100"
vglusterd $topwd 101
TEST $VCLI peer probe 127.0.0.101
vglusterd $topwd 102
TEST $VCLI peer probe 127.0.0.102
EXPECT_WITHIN 20 2 check_peers
create_cmd="$VCLI volume create $V0"
for i in $(seq 100 102); do
mkdir -p $B0/$V0$i
create_cmd="$create_cmd 127.0.0.$i:$B0/$V0$i"
done
TEST $create_cmd
TEST $VCLI volume set $V0 cluster.server-quorum-type server
TEST $VCLI volume start $V0
TEST glusterfs --volfile-server=127.0.0.100 --volfile-id=$V0 $M0
# Kill one pseudo-node, make sure the others survive and volume stays up.
kill -9 $(ps -ef | grep gluster | grep 127.0.0.102 | awk '{print $2}')
EXPECT_WITHIN 20 1 check_peers
fs_status=$(check_fs $M0)
nnodes=$(pidof glusterfsd | wc -w)
TEST [ "$fs_status" = 0 -a "$nnodes" = 2 ]
# Kill another pseudo-node, make sure the last one dies and volume goes down.
kill -9 $(ps -ef | grep gluster | grep 127.0.0.101 | awk '{print $2}')
EXPECT_WITHIN 20 0 check_peers
fs_status=$(check_fs $M0)
nnodes=$(pidof glusterfsd | wc -w)
TEST [ "$fs_status" = 1 -a "$nnodes" = 0 ]
cleanup

26
xlators/mgmt/glusterd/src/glusterd-handler.c
View File

@ -707,6 +707,7 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req)
glusterd_peerinfo_t *peerinfo = NULL;
gf_boolean_t run_fsm = _gf_true;
xlator_t *this = NULL;
char *bind_name = NULL;
GF_ASSERT (req);
this = THIS;
@ -736,7 +737,16 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req)
gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d",
cli_req.hostname, cli_req.port);
if (glusterd_is_local_addr(cli_req.hostname)) {
if (dict_get_str(this->options,"transport.socket.bind-address",
&bind_name) == 0) {
gf_log ("glusterd", GF_LOG_DEBUG,
"only checking probe address vs. bind address");
ret = glusterd_is_same_address(bind_name,cli_req.hostname);
}
else {
ret = glusterd_is_local_addr(cli_req.hostname);
}
if (ret) {
glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST, NULL,
cli_req.hostname, cli_req.port);
ret = 0;
@ -2434,6 +2444,7 @@ glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
dict_t *options = NULL;
int ret = -1;
glusterd_peerctx_t *peerctx = NULL;
data_t *data = NULL;
peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
if (!peerctx)
@ -2450,6 +2461,19 @@ glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
if (ret)
goto out;
/*
* For simulated multi-node testing, we need to make sure that we
* create our RPC endpoint with the same address that the peer would
* use to reach us.
*/
if (this->options) {
data = dict_get(this->options,"transport.socket.bind-address");
if (data) {
ret = dict_set(options,
"transport.socket.source-addr",data);
}
}
ret = glusterd_rpc_create (&peerinfo->rpc, options,
glusterd_peer_rpc_notify, peerctx);
if (ret) {

48
xlators/mgmt/glusterd/src/glusterd-utils.c
View File

@ -7396,3 +7396,51 @@ glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key)
return 0;
}
gf_boolean_t
glusterd_is_same_address (char *name1, char *name2)
{
struct addrinfo *addr1 = NULL;
struct addrinfo *addr2 = NULL;
struct addrinfo *p = NULL;
struct addrinfo *q = NULL;
gf_boolean_t ret = _gf_false;
int gai_err = 0;
gai_err = getaddrinfo(name1,NULL,NULL,&addr1);
if (gai_err != 0) {
gf_log (name1, GF_LOG_WARNING,
"error in getaddrinfo: %s\n", gai_strerror(gai_err));
goto out;
}
gai_err = getaddrinfo(name2,NULL,NULL,&addr2);
if (gai_err != 0) {
gf_log (name2, GF_LOG_WARNING,
"error in getaddrinfo: %s\n", gai_strerror(gai_err));
goto out;
}
for (p = addr1; p; p = p->ai_next) {
for (q = addr2; q; q = q->ai_next) {
if (p->ai_addrlen != q->ai_addrlen) {
continue;
}
if (memcmp(p->ai_addr,q->ai_addr,p->ai_addrlen)) {
continue;
}
ret = _gf_true;
goto out;
}
}
out:
if (addr1) {
freeaddrinfo(addr1);
}
if (addr2) {
freeaddrinfo(addr2);
}
return ret;
}

3
xlators/mgmt/glusterd/src/glusterd-utils.h
View File

@ -509,4 +509,7 @@ glusterd_generate_and_set_task_id (dict_t *dict, char *key);
int
glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
gf_boolean_t
glusterd_is_same_address (char *name1, char *name2);
#endif