andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Server backend storage hang should not cause the mount point to hang.

Browse Source 
Submitted-by: Krishna Srinivas <krishna@gluster.com>
NOTE: fixed compilation issues in posix.c introduced while merging

storage/posix polls for FS/kernel being functional by issuing
statvfs() call. In case statvfs expires the timer, storage/posix will
send CHILD_DOWN to upper translator. Ultimately this will cause
protocol/server to disconnect all clients connected and also cleans up
the data structures. Hence if soft lockup or other kernel bug causes
backend FS to hang, the clients will not be hung.

Signed-off-by: Krishna Srinivas <krishna@gluster.com>
Signed-off-by: Anand V. Avati <avati@blackhole.gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 272 (Server backend storage hang should not cause the mount point to hang)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=272
This commit is contained in:
Anand Avati 2010-01-23 05:14:31 +00:00 committed by Anand V. Avati
parent 9c53d5daf4
commit a0b148ea4e
7 changed files with 305 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libglusterfs/src/xlator.c
View File

@ -870,10 +870,6 @@ xlator_tree_init (xlator_t *xl)
*/
ret = xlator_init_rec (top);
if (ret == 0 && top->notify) {
top->notify (top, GF_EVENT_PARENT_UP, NULL);
}
return ret;
}

112
xlators/protocol/server/src/server-helpers.c
View File

@ -645,13 +645,14 @@ out:
int
server_connection_cleanup (xlator_t *this, server_connection_t *conn)
server_connection_cleanup (xlator_t *this, server_connection_t *conn, transport_t *trans)
{
char do_cleanup = 0;
struct _lock_table *ltable = NULL;
fdentry_t *fdentries = NULL;
uint32_t fd_count = 0;
int ret = 0;
int i = 0;
if (conn == NULL) {
goto out;
@ -659,6 +660,12 @@ server_connection_cleanup (xlator_t *this, server_connection_t *conn)
pthread_mutex_lock (&conn->lock);
{
for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++) {
if (conn->transports[i] == trans) {
conn->transports[i] = NULL;
transport_unref (trans);
}
}
conn->active_transports--;
if (conn->active_transports == 0) {
if (conn->ltable) {
@ -850,11 +857,12 @@ out:
server_connection_t *
server_connection_get (xlator_t *this, const char *id)
server_connection_get (xlator_t *this, const char *id, transport_t *trans)
{
server_connection_t *conn = NULL;
server_connection_t *trav = NULL;
server_conf_t *conf = NULL;
int i = 0;
conf = this->private;
@ -878,10 +886,30 @@ server_connection_get (xlator_t *this, const char *id)
list_add (&conn->list, &conf->conns);
}
if (conn->active_transports == TRANSPORTS_PER_SERVER_CONN) {
gf_log (this->name, GF_LOG_DEBUG,
"Maximum number of connections allowed is %d",
TRANSPORTS_PER_SERVER_CONN);
goto unlock;
}
for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++) {
if (!conn->transports[i])
break;
}
if (i == TRANSPORTS_PER_SERVER_CONN) {
gf_log (this->name, GF_LOG_DEBUG,
"Could not find a vacant slot");
goto unlock;
}
conn->transports[i] = transport_ref (trans);
conn->ref++;
conn->active_transports++;
}
unlock:
pthread_mutex_unlock (&conf->mutex);
return conn;
@ -918,3 +946,83 @@ server_connection_put (xlator_t *this, server_connection_t *conn)
out:
return;
}
void
server_child_down (xlator_t *this, xlator_t *bound_xl)
{
server_conf_t *conf = NULL;
server_connection_t *trav = NULL;
transport_t *trans = NULL;
int subvol_idx = 0;
int i = 0;
xlator_list_t *xltrav = NULL;
conf = this->private;
if (conf == NULL)
return;
xltrav = this->children;
while (xltrav) {
if (xltrav->xlator == bound_xl)
break;
xltrav = xltrav->next;
subvol_idx++;
}
gf_log (this->name, GF_LOG_DEBUG,
"subvolume %s(%d) went down", bound_xl->name, subvol_idx);
conf->subvol_list[subvol_idx] = 0;
pthread_mutex_lock (&conf->mutex);
{
if (!list_empty(&conf->conns)) {
list_for_each_entry (trav, &conf->conns, list) {
if (bound_xl == trav->bound_xl) {
gf_log (this->name, GF_LOG_DEBUG,
"disonnecting conn=%p", trav);
for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++)
{
trans = trav->transports[i];
if (trans == NULL)
continue;
gf_log (this->name, GF_LOG_DEBUG,
"disconnecting %p(%d)",
trans, i);
transport_disconnect (trans);
}
}
}
}
}
pthread_mutex_unlock (&conf->mutex);
}
void
server_child_up (xlator_t *this, xlator_t *bound_xl)
{
server_conf_t *conf = NULL;
int subvol_idx = 0;
xlator_list_t *xltrav = NULL;
conf = this->private;
if (conf == NULL)
return;
xltrav = this->children;
while (xltrav) {
if (bound_xl == xltrav->xlator) {
break;
}
subvol_idx++;
xltrav = xltrav->next;
}
gf_log (this->name, GF_LOG_DEBUG,
"subvolume %s(%d) came up", bound_xl->name, subvol_idx);
conf->subvol_list[subvol_idx] = 1;
}

6
xlators/protocol/server/src/server-helpers.h
View File

@ -69,4 +69,10 @@ gf_direntry_to_bin (dir_entry_t *head, char *bufferp);
void
server_print_request (call_frame_t *frame);
void
server_child_up (xlator_t *this, xlator_t *bound_xl);
void
server_child_down (xlator_t *this, xlator_t *bound_xl);
#endif /* __SERVER_HELPERS_H__ */

39
xlators/protocol/server/src/server-protocol.c
View File

@ -5654,6 +5654,8 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
char *volfile_key = NULL;
uint32_t checksum = 0;
int32_t lru_limit = 1024;
xlator_list_t *xltrav = NULL;
int subvol_idx = 0;
params = dict_new ();
reply = dict_new ();
@ -5695,7 +5697,7 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
}
conn = server_connection_get (frame->this, process_uuid);
conn = server_connection_get (frame->this, process_uuid, trans);
if (trans->xl_private != conn)
trans->xl_private = conn;
@ -5862,6 +5864,22 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
ret = dict_set_uint64 (reply, "transport-ptr",
((uint64_t) (long) trans));
xltrav = frame->this->children;
while (xltrav) {
if (xltrav->xlator == xl)
break;
xltrav = xltrav->next;
subvol_idx++;
}
if (conf->subvol_list[subvol_idx] == 0) {
gf_log (xl->name, GF_LOG_DEBUG,
"subvolume %d down (filesystem not accesible), failed to setvolume", subvol_idx);
op_ret = -1;
op_errno = ENOTCONN;
goto fail;
}
fail:
dict_len = dict_serialized_length (reply);
if (dict_len < 0) {
@ -6550,6 +6568,8 @@ init (xlator_t *this)
server_conf_t *conf = NULL;
data_t *data = NULL;
data_t *trace = NULL;
int i = 0;
xlator_list_t *xltrav = NULL;
if (this->children == NULL) {
gf_log (this->name, GF_LOG_ERROR,
@ -6635,6 +6655,15 @@ init (xlator_t *this)
}
}
xltrav = this->children;
while (xltrav) {
i++;
xltrav = xltrav->next;
}
conf->subvol_list = calloc (i, sizeof (char));
#ifndef GF_DARWIN_HOST_OS
{
struct rlimit lim;
@ -6736,6 +6765,12 @@ notify (xlator_t *this, int32_t event, void *data, ...)
}
switch (event) {
case GF_EVENT_CHILD_DOWN:
server_child_down (this, data);
break;
case GF_EVENT_CHILD_UP:
server_child_up (this, data);
break;
case GF_EVENT_POLLIN:
ret = protocol_server_pollin (this, trans);
break;
@ -6756,7 +6791,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
* FIXME: shouldn't we check for return value?
* what should be done if cleanup fails?
*/
server_connection_cleanup (this, trans->xl_private);
server_connection_cleanup (this, trans->xl_private, trans);
}
}
break;

7
xlators/protocol/server/src/server-protocol.h
View File

@ -37,6 +37,7 @@
#define DEFAULT_BLOCK_SIZE 4194304 /* 4MB */
#define DEFAULT_VOLUME_FILE_PATH CONFDIR "/glusterfs.vol"
#define TRANSPORTS_PER_SERVER_CONN 2
typedef struct _server_state server_state_t;
@ -64,6 +65,7 @@ struct _server_connection {
char *id;
int ref;
int active_transports;
transport_t *transports[TRANSPORTS_PER_SERVER_CONN];
pthread_mutex_t lock;
char disconnected;
fdtable_t *fdtable;
@ -75,7 +77,7 @@ typedef struct _server_connection server_connection_t;
server_connection_t *
server_connection_get (xlator_t *this, const char *id);
server_connection_get (xlator_t *this, const char *id, transport_t *trans);
void
server_connection_put (xlator_t *this, server_connection_t *conn);
@ -84,7 +86,7 @@ int
server_connection_destroy (xlator_t *this, server_connection_t *conn);
int
server_connection_cleanup (xlator_t *this, server_connection_t *conn);
server_connection_cleanup (xlator_t *this, server_connection_t *conn, transport_t *trans);
int
server_nop_cbk (call_frame_t *frame, void *cookie,
@ -107,6 +109,7 @@ typedef struct {
pthread_mutex_t mutex;
struct list_head conns;
gf_boolean_t verify_volfile_checksum;
char *subvol_list;
gf_boolean_t trace;
} server_conf_t;

141
xlators/storage/posix/src/posix.c
View File

@ -4703,6 +4703,119 @@ posix_inode (xlator_t *this)
return 0;
}
void
posix_fsping_timer_expired (void *data)
{
xlator_t *this = NULL;
struct posix_private *priv = NULL;
this = data;
priv = this->private;
pthread_mutex_lock (&priv->mutex);
{
if (priv->fsping_timer) {
gf_timer_call_cancel (this->ctx,
priv->fsping_timer);
priv->fsping_timer = NULL;
}
if (priv->fs_state) {
priv->fs_state = 0;
default_notify (this, GF_EVENT_CHILD_DOWN, NULL);
}
}
pthread_mutex_unlock (&priv->mutex);
}
void
posix_fsping (void *arg);
void *
posix_fsping_statvfs (void *arg)
{
int ret = -1;
xlator_t *this = NULL;
char *root_path = NULL;
struct statvfs buf = {0, };
struct posix_private *priv = NULL;
struct timeval delta = {0, };
this = arg;
priv = this->private;
root_path = POSIX_BASE_PATH (this);
ret = statvfs (root_path, &buf);
pthread_mutex_lock (&priv->mutex);
{
if (priv->fsping_timer) {
gf_timer_call_cancel (this->ctx,
priv->fsping_timer);
priv->fsping_timer = NULL;
}
if (ret == 0) {
if (priv->fs_state == 0) {
priv->fs_state = 1;
default_notify (this, GF_EVENT_CHILD_UP,
NULL);
}
} else {
if (priv->fs_state) {
priv->fs_state = 0;
default_notify (this, GF_EVENT_CHILD_DOWN,
NULL);
}
}
}
pthread_mutex_unlock (&priv->mutex);
delta.tv_sec = POSIX_FSPING_SLEEP_TIME;
priv->fsping_timer =
gf_timer_call_after (this->ctx,
delta,
posix_fsping,
(void *) this);
if (priv->fsping_timer == NULL) {
gf_log (this->name, GF_LOG_ERROR,
"unable to register timer");
}
return NULL;
}
void
posix_fsping (void *arg)
{
xlator_t *this = NULL;
struct posix_private *priv = NULL;
struct timeval delta = {0, };
this = arg;
priv = this->private;
delta.tv_sec = priv->fsping_timeout;
delta.tv_usec = 0;
if (priv->fsping_timer) {
gf_timer_call_cancel (this->ctx,
priv->fsping_timer);
}
priv->fsping_timer =
gf_timer_call_after (this->ctx,
delta,
posix_fsping_timer_expired,
(void *) this);
if (priv->fsping_timer == NULL) {
gf_log (this->name, GF_LOG_ERROR,
"unable to register timer");
/*FIXME: handle error*/
}
pthread_create (&priv->fsping,
NULL,
posix_fsping_statvfs,
this);
}
int32_t
posix_rchecksum (call_frame_t *frame, xlator_t *this,
@ -4780,12 +4893,15 @@ notify (xlator_t *this,
void *data,
...)
{
struct posix_private *priv = NULL;
priv = this->private;
switch (event)
{
case GF_EVENT_PARENT_UP:
{
/* Tell the parent that posix xlator is up */
default_notify (this, GF_EVENT_CHILD_UP, data);
posix_fsping ((void *)this);
}
break;
default:
@ -4809,9 +4925,9 @@ init (xlator_t *this)
data_t * dir_data = NULL;
data_t * tmp_data = NULL;
uint64_t time64 = 0;
int dict_ret = 0;
int32_t janitor_sleep;
int dict_ret = -1;
int fsping_timeout = -1;
int32_t janitor_sleep;
dir_data = dict_get (this->options, "directory");
@ -4908,6 +5024,7 @@ init (xlator_t *this)
strcat (_private->trash_path, "/" GF_REPLICATE_TRASH_DIR);
LOCK_INIT (&_private->lock);
pthread_mutex_init (&_private->mutex, NULL);
ret = gethostname (_private->hostname, 256);
if (ret < 0) {
@ -4923,6 +5040,17 @@ init (xlator_t *this)
_private->max_write = 1;
}
_private->fsping_timeout = POSIX_FSPING_TIMEOUT;
dict_ret = dict_get_int32 (this->options,
"fsping-timeout",
&fsping_timeout);
if (dict_ret == 0) {
_private->fsping_timeout = fsping_timeout;
}
gf_log (this->name, GF_LOG_DEBUG,
"fsping-timeout set to %d", _private->fsping_timeout);
_private->export_statfs = 1;
tmp_data = dict_get (this->options, "export-statfs-size");
if (tmp_data) {
@ -5056,6 +5184,7 @@ fini (xlator_t *this)
{
struct posix_private *priv = this->private;
sys_lremovexattr (priv->base_path, "trusted.glusterfs.test");
pthread_mutex_destroy (&priv->mutex);
FREE (priv);
return;
}
@ -5132,6 +5261,8 @@ struct volume_options options[] = {
.type = GF_OPTION_TYPE_BOOL },
{ .key = {"span-devices"},
.type = GF_OPTION_TYPE_INT },
{ .key = {"fsping-timeout"},
.type = GF_OPTION_TYPE_INT },
{ .key = {"background-unlink"},
.type = GF_OPTION_TYPE_BOOL },
{ .key = {"janitor-sleep-duration"},

11
xlators/storage/posix/src/posix.h
View File

@ -29,6 +29,7 @@
#include <unistd.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <time.h>
#ifdef linux
@ -50,6 +51,7 @@
#include "xlator.h"
#include "inode.h"
#include "compat.h"
#include "timer.h"
/**
* posix_fd - internal structure common to file and directory fd's
@ -70,6 +72,7 @@ struct posix_private {
int32_t base_path_length;
gf_lock_t lock;
pthread_mutex_t mutex;
char hostname[256];
/* Statistics, provides activity of the server */
@ -117,6 +120,11 @@ struct posix_private {
int num_devices_to_span;
dev_t *st_device;
pthread_t fsping;
gf_timer_t *fsping_timer;
int fsping_timeout;
int fs_state;
/* a global generation number sequence is used to assign generation numbers in
sequence.
*/
@ -129,6 +137,9 @@ struct posix_private {
char * trash_path;
};
#define POSIX_FSPING_SLEEP_TIME 10
#define POSIX_FSPING_TIMEOUT 10
#define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
#define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)