andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

socket: use TCP_USER_TIMEOUT to detect client failures quicker

Browse Source 
Use the network.ping-timeout to set the TCP_USER_TIMEOUT socket option
(see 'man 7 tcp'). The option sets the transport.tcp-user-timeout option
that is handled in the rpc/socket layer on the protocol/server side.
This socket option makes detecting unclean disconnected clients more
reliable.

When the socket gets closed, any locks that the client held are been
released. This makes it possible to reduce the fail-over time for
applications that run on systems that became unreachable due to
a network partition or general system error client-side (kernel panic,
hang, ...).

It is not trivial to create a test-case for this at the moment. We need
a client that unclean disconnects and an other client that tries to take
over the lock from the disconnected client.

URL: http://supercolony.gluster.org/pipermail/gluster-devel/2014-May/040755.html
Change-Id: I5e5f540a49abfb5f398291f1818583a63a5f4bb4
BUG: 1129787
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/8065
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: soumya k <skoduri@redhat.com>
Reviewed-by: Santosh Pradhan <santosh.pradhan@gmail.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
This commit is contained in:
Niels de Vos 2015-02-17 12:12:11 +01:00 committed by Kaleb KEITHLEY
parent 2b97b57cd8
commit 6b37049902
7 changed files with 76 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
rpc/rpc-lib/src/rpc-transport.c
View File

@ -572,7 +572,7 @@ out:
//why call it if you dont set it.
int
rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
int32_t time)
int32_t time, int32_t timeout)
{
int ret = -1;
@ -588,6 +588,11 @@ rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
"transport.socket.keepalive-time", time);
if (ret)
goto out;
ret = dict_set_int32 (options,
"transport.tcp-user-timeout", timeout);
if (ret)
goto out;
out:
return ret;
}

2
rpc/rpc-lib/src/rpc-transport.h
View File

@ -306,7 +306,7 @@ rpc_transport_pollin_destroy (rpc_transport_pollin_t *pollin);
int
rpc_transport_keepalive_options_set (dict_t *options, int32_t interval,
int32_t time);
int32_t time, int32_t timeout);
int
rpc_transport_unix_options_build (dict_t **options, char *filepath,

55
rpc/rpc-transport/socket/src/socket.c
View File

@ -31,9 +31,15 @@
#include "xdr-nfs3.h"
#include "rpcsvc.h"
/* for TCP_USER_TIMEOUT */
#if !defined(TCP_USER_TIMEOUT) && defined(GF_LINUX_HOST_OS)
#include <linux/tcp.h>
#else
#include <netinet/tcp.h>
#endif
#include <fcntl.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <rpc/xdr.h>
#include <sys/ioctl.h>
#define GF_LOG_ERRNO(errno) ((errno == ENOTCONN) ? GF_LOG_DEBUG : GF_LOG_ERROR)
@ -857,10 +863,12 @@ __socket_nodelay (int fd)
static int
__socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
__socket_keepalive (int fd, int family, int keepalive_intvl,
int keepalive_idle, int timeout)
{
int on = 1;
int ret = -1;
int timeout_ms = timeout * 1000;
ret = setsockopt (fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof (on));
if (ret == -1) {
@ -890,7 +898,7 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
goto done;
ret = setsockopt (fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepalive_idle,
sizeof (keepalive_intvl));
sizeof (keepalive_idle));
if (ret == -1) {
gf_log ("socket", GF_LOG_WARNING,
"failed to set keep idle %d on socket %d, %s",
@ -905,11 +913,23 @@ __socket_keepalive (int fd, int family, int keepalive_intvl, int keepalive_idle)
keepalive_intvl, fd, strerror(errno));
goto err;
}
#if defined(TCP_USER_TIMEOUT)
ret = setsockopt (fd, IPPROTO_TCP , TCP_USER_TIMEOUT, &timeout_ms,
sizeof (timeout_ms));
if (ret == -1) {
gf_log ("socket", GF_LOG_WARNING, "failed to set "
"TCP_USER_TIMEOUT %d on socket %d, %s", timeout_ms, fd,
strerror(errno));
goto err;
}
#endif
#endif
done:
gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, interval "
"%d, idle: %d", fd, keepalive_intvl, keepalive_idle);
gf_log (THIS->name, GF_LOG_TRACE, "Keep-alive enabled for socket %d, "
"interval %d, idle: %d, timeout: %d", fd, keepalive_intvl,
keepalive_idle, timeout);
err:
return ret;
@ -2642,7 +2662,8 @@ socket_server_event_handler (int fd, int idx, void *data,
ret = __socket_keepalive (new_sock,
new_sockaddr.ss_family,
priv->keepaliveintvl,
priv->keepaliveidle);
priv->keepaliveidle,
priv->timeout);
if (ret == -1)
gf_log (this->name, GF_LOG_WARNING,
"Failed to set keep-alive: %s",
@ -2986,7 +3007,8 @@ socket_connect (rpc_transport_t *this, int port)
ret = __socket_keepalive (priv->sock,
sa_family,
priv->keepaliveintvl,
priv->keepaliveidle);
priv->keepaliveidle,
priv->timeout);
if (ret == -1)
gf_log (this->name, GF_LOG_ERROR,
"Failed to set keep-alive: %s",
@ -3577,6 +3599,7 @@ reconfigure (rpc_transport_t *this, dict_t *options)
char *optstr = NULL;
int ret = 0;
uint64_t windowsize = 0;
uint32_t timeout = 0;
GF_VALIDATE_OR_GOTO ("socket", this, out);
GF_VALIDATE_OR_GOTO ("socket", this->private, out);
@ -3605,6 +3628,13 @@ reconfigure (rpc_transport_t *this, dict_t *options)
else
priv->keepalive = 1;
if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
&timeout) == 0) {
priv->timeout = timeout;
gf_log (this->name, GF_LOG_DEBUG, "Reconfigued "
"transport.tcp-user-timeout=%d", timeout);
}
optstr = NULL;
if (dict_get_str (this->options, "tcp-window-size",
&optstr) == 0) {
@ -3659,6 +3689,7 @@ socket_init (rpc_transport_t *this)
uint64_t windowsize = GF_DEFAULT_SOCKET_WINDOW_SIZE;
char *optstr = NULL;
uint32_t keepalive = 0;
uint32_t timeout = 0;
uint32_t backlog = 0;
int session_id = 0;
int32_t cert_depth = 1;
@ -3771,6 +3802,13 @@ socket_init (rpc_transport_t *this)
priv->keepaliveidle = keepalive;
}
if (dict_get_uint32 (this->options, "transport.tcp-user-timeout",
&timeout) == 0) {
priv->timeout = timeout;
}
gf_log (this->name, GF_LOG_DEBUG, "Configued "
"transport.tcp-user-timeout=%d", priv->timeout);
if (dict_get_uint32 (this->options,
"transport.socket.listen-backlog",
&backlog) == 0) {
@ -4027,6 +4065,9 @@ struct volume_options options[] = {
.min = GF_MIN_SOCKET_WINDOW_SIZE,
.max = GF_MAX_SOCKET_WINDOW_SIZE
},
{ .key = {"transport.tcp-user-timeout"},
.type = GF_OPTION_TYPE_INT,
},
{ .key = {"transport.socket.nodelay"},
.type = GF_OPTION_TYPE_BOOL
},

1
rpc/rpc-transport/socket/src/socket.h
View File

@ -216,6 +216,7 @@ typedef struct {
int keepalive;
int keepaliveidle;
int keepaliveintvl;
int timeout;
uint32_t backlog;
gf_boolean_t read_fail_log;
gf_boolean_t ssl_enabled; /* outbound I/O */

12
xlators/mgmt/glusterd/src/glusterd-handler.c
View File

@ -2994,7 +2994,8 @@ out:
}
int
glusterd_transport_keepalive_options_get (int *interval, int *time)
glusterd_transport_keepalive_options_get (int *interval, int *time,
int *timeout)
{
int ret = 0;
xlator_t *this = NULL;
@ -3008,6 +3009,9 @@ glusterd_transport_keepalive_options_get (int *interval, int *time)
ret = dict_get_int32 (this->options,
"transport.socket.keepalive-time",
time);
ret = dict_get_int32 (this->options,
"transport.tcp-user-timeout",
timeout);
return 0;
}
@ -3018,6 +3022,7 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
dict_t *dict = NULL;
int32_t interval = -1;
int32_t time = -1;
int32_t timeout = -1;
int ret = 0;
GF_ASSERT (options);
@ -3044,10 +3049,11 @@ glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
}
/* Set keepalive options */
glusterd_transport_keepalive_options_get (&interval, &time);
glusterd_transport_keepalive_options_get (&interval, &time, &timeout);
if ((interval > 0) || (time > 0))
ret = rpc_transport_keepalive_options_set (dict, interval, time);
ret = rpc_transport_keepalive_options_set (dict, interval,
time, timeout);
*options = dict;
out:
gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);

5
xlators/mgmt/glusterd/src/glusterd-volume-set.c
View File

@ -842,6 +842,11 @@ struct volopt_map_entry glusterd_volopt_map[] = {
},
/* Server xlator options */
{ .key = "network.ping-timeout",
.voltype = "protocol/server",
.option = "transport.tcp-user-timeout",
.op_version = GD_OP_VERSION_3_7_0,
},
{ .key = "network.tcp-window-size",
.voltype = "protocol/server",
.op_version = 1

6
xlators/protocol/server/src/server.c
View File

@ -1198,6 +1198,12 @@ struct volume_options options[] = {
{ .key = {"volume-filename.*"},
.type = GF_OPTION_TYPE_PATH,
},
{ .key = {"transport.tcp-user-timeout"},
.type = GF_OPTION_TYPE_TIME,
.min = 0,
.max = 1013,
.default_value = "42", /* default like network.ping-timeout */
},
{ .key = {"transport.*"},
.type = GF_OPTION_TYPE_ANY,
},