andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

gNFS: Set default outstanding RPC limit to 16

Browse Source 
With 64, NFS server hangs with large I/O load (~ 64 threads writing
to NFS server). The test results from Ben England (Performance expert)
suggest to set it as 16 instead of 64.

Change-Id: I418ff5ba0a3e9fdb14f395b8736438ee1bbd95f4
BUG: 1008301
Signed-off-by: Santosh Kumar Pradhan <spradhan@redhat.com>
Reviewed-on: http://review.gluster.org/6696
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: ben england <bengland@redhat.com>
This commit is contained in:
Santosh Kumar Pradhan 2014-01-14 11:21:44 +05:30 committed by Anand Avati
parent f3e227d525
commit 260e817b8a
4 changed files with 67 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
rpc/rpc-lib/src/rpcsvc.c
View File

@ -1883,8 +1883,7 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options)
if (!svc->register_portmap)
gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration "
"disabled");
ret = rpcsvc_set_outstanding_rpc_limit (svc, options);
ret = 0;
out:
return ret;
}
@ -2016,10 +2015,15 @@ out:
}
/*
* Reconfigure() the rpc.outstanding-rpc-limit param.
* Configure() the rpc.outstanding-rpc-limit param.
* If dict_get_int32() for dict-key "rpc.outstanding-rpc-limit" FAILS,
* it would set the value as "defvalue". Otherwise it would fetch the
* value and round up to multiple-of-8. defvalue must be +ve.
*
* NB: defval or set-value "0" is special which means unlimited/65536.
*/
int
rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue)
{
int ret = -1; /* FAILURE */
int rpclim = 0;
@ -2028,30 +2032,30 @@ rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
if ((!svc) || (!options))
return (-1);
/* Reconfigure() the rpc.outstanding-rpc-limit param */
if ((defvalue < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) ||
(defvalue > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT)) {
return (-1);
}
/* Fetch the rpc.outstanding-rpc-limit from dict. */
ret = dict_get_int32 (options, rpclimkey, &rpclim);
if (ret < 0) {
/* Fall back to default for FAILURE */
rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
} else {
/* SUCCESS: round off to multiple of 8.
* If the input value fails Boundary check, fall back to
* default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT.
* NB: value 0 is special, means its unset i.e. unlimited.
rpclim = defvalue;
}
/* Round up to multiple-of-8. It must not exceed
* RPCSVC_MAX_OUTSTANDING_RPC_LIMIT.
*/
rpclim = ((rpclim + 8 - 1) >> 3) * 8;
if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) {
rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
} else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
}
}
if (svc->outstanding_rpc_limit != rpclim) {
svc->outstanding_rpc_limit = rpclim;
gf_log (GF_RPCSVC, GF_LOG_INFO,
"Configured %s with value %d",
rpclimkey, rpclim);
"Configured %s with value %d", rpclimkey, rpclim);
}
return (0);
@ -2066,7 +2070,7 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
rpcsvc_t *svc = NULL;
int ret = -1;
if ((!ctx) || (!options))
if ((!xl) || (!ctx) || (!options))
return NULL;
svc = GF_CALLOC (1, sizeof (*svc), gf_common_mt_rpcsvc_t);

5
rpc/rpc-lib/src/rpcsvc.h
View File

@ -38,7 +38,8 @@
#define MAX_IOVEC 16
#endif
#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64
#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 /* Default for protocol/server */
#define RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT 16 /* Default for nfs/server */
#define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536
#define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */
@ -597,7 +598,7 @@ rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options);
int
rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options);
int
rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options);
rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue);
int
rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
rpcsvc_vector_sizer

22
xlators/nfs/server/src/nfs.c
View File

@ -967,6 +967,15 @@ nfs_init_state (xlator_t *this)
goto free_foppool;
}
ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
this->options,
RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
if (ret < 0) {
gf_log (GF_NFS, GF_LOG_ERROR,
"Failed to configure outstanding-rpc-limit");
goto free_foppool;
}
nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
this->private = (void *)nfs;
@ -1210,6 +1219,17 @@ reconfigure (xlator_t *this, dict_t *options)
"rpcsvc reconfigure options failed");
return (-1);
}
/* Reconfigure rpc.outstanding-rpc-limit */
ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
options,
RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
if (ret < 0) {
gf_log (GF_NFS, GF_LOG_ERROR,
"Failed to reconfigure outstanding-rpc-limit");
return (-1);
}
regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
if (nfs->register_portmap != regpmap) {
nfs->register_portmap = regpmap;
@ -1741,7 +1761,7 @@ struct volume_options options[] = {
.type = GF_OPTION_TYPE_INT,
.min = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
.max = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
.default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT),
.default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT),
.description = "Parameter to throttle the number of incoming RPC "
"requests from a client. 0 means no limit (can "
"potentially run out of memory)"

20
xlators/protocol/server/src/server.c
View File

@ -746,7 +746,15 @@ reconfigure (xlator_t *this, dict_t *options)
(void) rpcsvc_set_allow_insecure (rpc_conf, options);
(void) rpcsvc_set_root_squash (rpc_conf, options);
(void) rpcsvc_set_outstanding_rpc_limit (rpc_conf, options);
ret = rpcsvc_set_outstanding_rpc_limit (rpc_conf, options,
RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
if (ret < 0) {
gf_log (this->name, GF_LOG_ERROR,
"Failed to reconfigure outstanding-rpc-limit");
goto out;
}
list_for_each_entry (listeners, &(rpc_conf->listeners), list) {
if (listeners->trans != NULL) {
if (listeners->trans->reconfigure )
@ -860,12 +868,20 @@ init (xlator_t *this)
/* RPC related */
conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0);
if (conf->rpc == NULL) {
gf_log (this->name, GF_LOG_WARNING,
gf_log (this->name, GF_LOG_ERROR,
"creation of rpcsvc failed");
ret = -1;
goto out;
}
ret = rpcsvc_set_outstanding_rpc_limit (conf->rpc, this->options,
RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT);
if (ret < 0) {
gf_log (this->name, GF_LOG_ERROR,
"Failed to configure outstanding-rpc-limit");
goto out;
}
ret = rpcsvc_create_listeners (conf->rpc, this->options,
this->name);
if (ret < 1) {