andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

dht: "replica.split-brain-status" attribute value is not correct

Browse Source 
Problem: In a distributed-replicate volume attribute
         "replica.split-brain-status" value does not display split-brain
           condition though directory is in split-brain.
         If directory is in split brain on mutiple replica-pairs
         it does not show full list of replica pairs.

Solution: Update the dht_aggregate code to aggregate the xattr
          value in this specific condition.

Fix:      1) function getChoices returns the choices from split-brain
             status string.
          2) function add_opt adding the choices to local buffer to
             store in dictionary
          3) For the key "replica.split-brain-status" function dht_aggregate
             call dht_aggregate_split_brain_xattr to prepare the list.

Test:     To verify the patch followed below steps
          1) Create a distributed replica volume and create mount point
          2) Stop heal daemon
          3) Touch file and directories on mount point
             mkdir test{1..5};touch tmp{1..5}
          4) Down brick process on one of the replica set
             pkill -9 glusterfsd
          5) Change permission of dir on mount point
             chmod 755 test{1..5}
          6) Restart brick process on node with force option
          7) kill brick process on other node in same replica set
          8) Change permission of dir again on mount point
             chmod 766 test{1..5}
          9) Reexecute same step from 4-9 on other replica set also
          10) After check heal status on server it will show dir's are
              in split brain on all replica sets
          11) After check the replica.split-brain-status attr on mount
              point it will show wrong status of split brain.
          12) After apply the patch the attribute shows correct value.

BUG: 1368312
Change-Id: Icdfd72005a4aa82337c342762775a3d1761bbe4a
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
Reviewed-on: http://review.gluster.org/15201
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
This commit is contained in:
Mohit Agrawal 2016-08-19 10:33:50 +05:30 committed by Raghavendra G
parent 801cd07a4c
commit c4e9ec653c
3 changed files with 294 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
tests/bugs/bug-1368312.t Normal file
View File

@ -0,0 +1,84 @@
#!/bin/bash
. $(dirname $0)/../include.rc
. $(dirname $0)/../volume.rc
cleanup;
function compare_get_split_brain_status {
local path=$1
local choice=$2
echo `getfattr -n replica.split-brain-status $path` | cut -f2 -d"=" | sed -e 's/^"//' -e 's/"$//' | grep $choice
if [ $? -ne 0 ]
then
echo 1
else
echo 0
fi
}
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1,2,3,4,5}
TEST $CLI volume start $V0
#Disable self-heal-daemon
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0;
TEST mkdir $M0/tmp1
#Create metadata split-brain
TEST kill_brick $V0 $H0 $B0/${V0}0
TEST chmod 666 $M0/tmp1
TEST $CLI volume start $V0 force
TEST kill_brick $V0 $H0 $B0/${V0}1
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
TEST chmod 757 $M0/tmp1
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
EXPECT 2 get_pending_heal_count $V0
TEST kill_brick $V0 $H0 $B0/${V0}2
TEST chmod 755 $M0/tmp1
TEST $CLI volume start $V0 force
TEST kill_brick $V0 $H0 $B0/${V0}3
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
TEST chmod 766 $M0/tmp1
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3
EXPECT 4 get_pending_heal_count $V0
TEST kill_brick $V0 $H0 $B0/${V0}4
TEST chmod 765 $M0/tmp1
TEST $CLI volume start $V0 force
TEST kill_brick $V0 $H0 $B0/${V0}5
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4
TEST chmod 756 $M0/tmp1
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 4
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 5
EXPECT 6 get_pending_heal_count $V0
cd $M0
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-0
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-1
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-2
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-3
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-4
EXPECT 0 compare_get_split_brain_status ./tmp1 patchy-client-5
cd -
cleanup

212
xlators/cluster/dht/src/dht-common.c
View File

@ -127,6 +127,190 @@ out:
}
int add_opt(char **optsp, const char *opt)
{
char *newopts = NULL;
unsigned oldsize = 0;
unsigned newsize = 0;
if (*optsp == NULL)
newopts = gf_strdup (opt);
else {
oldsize = strlen (*optsp);
newsize = oldsize + 1 + strlen (opt) + 1;
newopts = GF_REALLOC (*optsp, newsize);
if (newopts)
sprintf (newopts + oldsize, ",%s", opt);
}
if (newopts == NULL) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to add choices in buffer in add_opt");
return -1;
}
*optsp = newopts;
return 0;
}
/* Return Choice list from Split brain status */
char *
getChoices (const char *value)
{
int i = 0;
char *ptr = NULL;
char *tok = NULL;
char *result = NULL;
char *newval = NULL;
ptr = strstr (value, "Choices:");
if (!ptr) {
result = ptr;
goto out;
}
newval = gf_strdup (ptr);
if (!newval) {
result = newval;
goto out;
}
tok = strtok (newval, ":");
if (!tok) {
result = tok;
goto out;
}
while (tok) {
i++;
if (i == 2)
break;
tok = strtok (NULL, ":");
}
result = gf_strdup (tok);
out:
if (newval)
GF_FREE (newval);
return result;
}
/* This function prepare a list of choices for key
(replica.split-brain-status) in case of metadata split brain
only on the basis of key-value passed to this function.
After prepare the list of choices it update the same key in dict
with this value to reflect the same in
replica.split-brain-status attr for file.
*/
int
dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value)
{
int ret = 0;
char *oldvalue = NULL;
char *old_choice = NULL;
char *new_choice = NULL;
char *full_choice = NULL;
char *status = NULL;
if (value == NULL) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_DATA_NULL,
"GF_AFR_SBRAIN_STATUS value is NULL");
ret = -1;
goto out;
}
ret = dict_get_str (dst, key, &oldvalue);
if (ret)
goto out;
if (oldvalue && (strstr (oldvalue, "not"))) {
gf_msg_debug ("dht", 0,
"Need to update split-brain status in dict");
ret = -1;
goto out;
}
if (oldvalue && (strstr (oldvalue, "metadata-split-brain:yes"))
&& (strstr (oldvalue, "data-split-brain:no"))) {
if (strstr (value->data, "not")) {
gf_msg_debug ("dht", 0,
"No need to update split-brain status");
ret = 0;
goto out;
}
if (strstr (value->data, "yes") &&
(strncmp (oldvalue, value->data, strlen(oldvalue)))) {
old_choice = getChoices (oldvalue);
if (!old_choice) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to get choices");
ret = -1;
goto out;
}
ret = add_opt (&full_choice, old_choice);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to add choices");
ret = -1;
goto out;
}
new_choice = getChoices (value->data);
if (!new_choice) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to get choices");
ret = -1;
goto out;
}
ret = add_opt (&full_choice, new_choice);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to add choices ");
ret = -1;
goto out;
}
ret = gf_asprintf (&status,
"data-split-brain:%s "
"metadata-split-brain:%s Choices:%s",
"no", "yes", full_choice);
if (-1 == ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_NO_MEMORY,
"Error to prepare status ");
goto out;
}
ret = dict_set_dynstr (dst, key, status);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_DICT_SET_FAILED,
"Failed to set full choice");
}
}
}
out:
if (old_choice)
GF_FREE (old_choice);
if (new_choice)
GF_FREE (new_choice);
if (full_choice)
GF_FREE (full_choice);
return ret;
}
int
dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
@ -137,18 +321,22 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
dst = data;
if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
/* compare split brain xattr only */
if (strcmp (key, GF_AFR_SBRAIN_STATUS) == 0) {
ret = dht_aggregate_split_brain_xattr(dst, key, value);
if (!ret)
goto out;
} else if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
ret = dht_aggregate_quota_xattr (dst, key, value);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
"Failed to aggregate quota xattr");
goto out;
}
goto out;
} else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
ret = gf_get_min_stime (THIS, dst, key, value);
if (ret < 0)
goto out;
goto out;
} else {
/* compare user xattrs only */
if (!strncmp (key, "user.", strlen ("user."))) {
@ -161,16 +349,16 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
key);
}
}
ret = dict_set (dst, key, value);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_DICT_SET_FAILED,
"Failed to set dictionary value: key = %s",
key);
}
}
ret = 0;
ret = dict_set (dst, key, value);
if (ret) {
gf_msg ("dht", GF_LOG_WARNING, 0,
DHT_MSG_DICT_SET_FAILED,
"Failed to set dictionary value: key = %s",
key);
}
out:
return ret;
}

10
xlators/cluster/dht/src/dht-common.h
View File

@ -1269,4 +1269,14 @@ void
dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
unsigned long frsize);
int
add_opt(char **optsp, const char *opt);
char *
getChoices (const char *value);
int
dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value);
#endif/* _DHT_H */