andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

protocol: Remove lock recovery logic from client and server

Browse Source 
Change-Id: I27f5e1e34fe3eac96c7dd88e90753fb5d3d14550
BUG: 1272030
Signed-off-by: Anoop C S <anoopcs@redhat.com>
This commit is contained in:
Anoop C S 2015-10-15 15:04:34 +05:30 committed by Raghavendra G
parent 596143a286
commit 3e78ea991b
11 changed files with 36 additions and 924 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
xlators/mgmt/glusterd/src/glusterd-volume-set.c
View File

@ -1951,18 +1951,6 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.op_version = 1,
.flags = VOLOPT_FLAG_CLIENT_OPT
},
{ .key = "features.lock-heal",
.voltype = "protocol/client",
.option = "lk-heal",
.op_version = 1,
.flags = VOLOPT_FLAG_CLIENT_OPT
},
{ .key = "features.grace-timeout",
.voltype = "protocol/client",
.option = "grace-timeout",
.op_version = 1,
.flags = VOLOPT_FLAG_CLIENT_OPT
},
{ .key = "client.ssl",
.voltype = "protocol/client",
.option = "transport.socket.ssl-enabled",
@ -2076,18 +2064,6 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.type = GLOBAL_DOC,
.op_version = 3
},
{ .key = "features.lock-heal",
.voltype = "protocol/server",
.option = "lk-heal",
.type = NO_DOC,
.op_version = 1
},
{ .key = "features.grace-timeout",
.voltype = "protocol/server",
.option = "grace-timeout",
.type = NO_DOC,
.op_version = 1
},
{ .key = "server.ssl",
.voltype = "protocol/server",
.option = "transport.socket.ssl-enabled",

365
xlators/protocol/client/src/client-handshake.c
View File

@ -27,11 +27,6 @@ extern rpc_clnt_prog_t clnt3_3_fop_prog;
extern rpc_clnt_prog_t clnt4_0_fop_prog;
extern rpc_clnt_prog_t clnt_pmap_prog;
int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe);
int client_set_lk_version (xlator_t *this);
typedef struct client_fd_lk_local {
gf_atomic_t ref;
gf_boolean_t error;
@ -168,7 +163,6 @@ clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx,
pthread_spin_lock (&conf->fd_lock);
{
fdctx->remote_fd = -1;
fdctx->lk_heal_state = GF_LK_HEAL_DONE;
}
pthread_spin_unlock (&conf->fd_lock);
@ -177,90 +171,6 @@ out:
return ret;
}
int
client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int32_t ret = -1;
call_frame_t *fr = NULL;
gf_set_lk_ver_rsp rsp = {0,};
fr = (call_frame_t *) myframe;
GF_VALIDATE_OR_GOTO ("client", fr, out);
if (req->rpc_status == -1) {
gf_msg (fr->this->name, GF_LOG_WARNING, ENOTCONN,
PC_MSG_RPC_STATUS_ERROR, "received RPC status error");
goto out;
}
ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp);
if (ret < 0)
gf_msg (fr->this->name, GF_LOG_WARNING, 0,
PC_MSG_XDR_DECODING_FAILED, "xdr decoding failed");
else
gf_msg (fr->this->name, GF_LOG_INFO, 0,
PC_MSG_LOCK_VERSION_SERVER,
"Server lk version = %d", rsp.lk_ver);
ret = 0;
out:
if (fr)
STACK_DESTROY (fr->root);
return ret;
}
//TODO: Check for all released fdctx and destroy them
int
client_set_lk_version (xlator_t *this)
{
int ret = -1;
clnt_conf_t *conf = NULL;
call_frame_t *frame = NULL;
gf_set_lk_ver_req req = {0, };
char *process_uuid = NULL;
GF_VALIDATE_OR_GOTO ("client", this, err);
conf = (clnt_conf_t *) this->private;
req.lk_ver = client_get_lk_ver (conf);
ret = dict_get_str (this->options, "process-uuid", &process_uuid);
if (!process_uuid) {
ret = -1;
goto err;
}
req.uid = gf_strdup (process_uuid);
if (!req.uid) {
ret = -1;
goto err;
}
frame = create_frame (this, this->ctx->pool);
if (!frame) {
ret = -1;
goto out;
}
gf_msg_debug (this->name, 0, "Sending SET_LK_VERSION");
ret = client_submit_request (this, &req, frame,
conf->handshake,
GF_HNDSK_SET_LK_VER,
client_set_lk_version_cbk,
NULL, NULL, 0, NULL, 0, NULL,
(xdrproc_t)xdr_gf_set_lk_ver_req);
out:
GF_FREE (req.uid);
return ret;
err:
gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_SET_LK_VERSION_ERROR,
"Failed to send SET_LK_VERSION to server");
return ret;
}
int
client_fd_lk_count (fd_lk_ctx_t *lk_ctx)
{
@ -440,186 +350,6 @@ out:
return ret;
}
int
client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
{
int32_t ret = -1;
xlator_t *this = NULL;
gfs3_lk_rsp rsp = {0,};
call_frame_t *frame = NULL;
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
clnt_fd_lk_local_t *local = NULL;
struct gf_flock lock = {0,};
frame = (call_frame_t *) myframe;
this = frame->this;
local = (clnt_fd_lk_local_t *) frame->local;
conf = (clnt_conf_t *) this->private;
if (req->rpc_status == -1) {
gf_msg ("client", GF_LOG_WARNING, 0, PC_MSG_CLIENT_REQ_FAIL,
"request failed at rpc");
goto out;
}
ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
if (ret < 0) {
gf_msg (this->name, GF_LOG_ERROR, EINVAL,
PC_MSG_XDR_DECODING_FAILED, "XDR decoding failed");
goto out;
}
if (rsp.op_ret == -1) {
gf_msg (this->name, GF_LOG_ERROR, 0, PC_MSG_LOCK_REQ_FAIL,
"lock request failed");
ret = -1;
goto out;
}
fdctx = local->fdctx;
gf_proto_flock_to_flock (&rsp.flock, &lock);
gf_msg_debug (this->name, 0, "%s type lock reacquired on file "
"with gfid %s from %"PRIu64 " to %"PRIu64,
get_lk_type (lock.l_type), uuid_utoa (fdctx->gfid),
lock.l_start, lock.l_start + lock.l_len);
if (!clnt_fd_lk_local_error_status (this, local) &&
clnt_fd_lk_local_unref (this, local) == 0) {
pthread_spin_lock (&conf->fd_lock);
{
fdctx->lk_heal_state = GF_LK_HEAL_DONE;
}
pthread_spin_unlock (&conf->fd_lock);
fdctx->reopen_done (fdctx, fdctx->remote_fd, this);
}
ret = 0;
out:
if (ret < 0) {
clnt_fd_lk_local_mark_error (this, local);
clnt_fd_lk_local_unref (this, local);
}
frame->local = NULL;
STACK_DESTROY (frame->root);
return ret;
}
int
_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
{
int32_t ret = -1;
int32_t gf_cmd = 0;
int32_t gf_type = 0;
gfs3_lk_req req = {{0,},};
struct gf_flock flock = {0,};
fd_lk_ctx_t *lk_ctx = NULL;
clnt_fd_lk_local_t *local = NULL;
fd_lk_ctx_node_t *fd_lk = NULL;
call_frame_t *frame = NULL;
clnt_conf_t *conf = NULL;
conf = (clnt_conf_t *) this->private;
lk_ctx = fdctx->lk_ctx;
local = clnt_fd_lk_local_create (fdctx);
if (!local) {
gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_LOCK_ERROR,
"clnt_fd_lk_local_create failed, aborting reacquring "
"of locks on %s.", uuid_utoa (fdctx->gfid));
clnt_reacquire_lock_error (this, fdctx, conf);
goto out;
}
list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) {
memcpy (&flock, &fd_lk->user_flock,
sizeof (struct gf_flock));
/* Always send F_SETLK even if the cmd was F_SETLKW */
/* to avoid frame being blocked if lock cannot be granted. */
ret = client_cmd_to_gf_cmd (F_SETLK, &gf_cmd);
if (ret) {
gf_msg (this->name, GF_LOG_WARNING, 0,
PC_MSG_LOCK_ERROR, "client_cmd_to_gf_cmd "
"failed, aborting reacquiring of locks");
break;
}
gf_type = client_type_to_gf_type (flock.l_type);
req.fd = fdctx->remote_fd;
req.cmd = gf_cmd;
req.type = gf_type;
(void) gf_proto_flock_from_flock (&req.flock,
&flock);
memcpy (req.gfid, fdctx->gfid, 16);
frame = create_frame (this, this->ctx->pool);
if (!frame) {
ret = -1;
break;
}
frame->local = clnt_fd_lk_local_ref (this, local);
frame->root->lk_owner = fd_lk->user_flock.l_owner;
ret = client_submit_request (this, &req, frame,
conf->fops, GFS3_OP_LK,
client_reacquire_lock_cbk,
NULL, NULL, 0, NULL, 0, NULL,
(xdrproc_t)xdr_gfs3_lk_req);
if (ret) {
gf_msg (this->name, GF_LOG_WARNING, 0,
PC_MSG_LOCK_REACQUIRE, "reacquiring locks "
"failed on file with gfid %s",
uuid_utoa (fdctx->gfid));
break;
}
ret = 0;
frame = NULL;
}
if (local)
(void) clnt_fd_lk_local_unref (this, local);
out:
return ret;
}
int
client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
{
int32_t ret = -1;
fd_lk_ctx_t *lk_ctx = NULL;
GF_VALIDATE_OR_GOTO ("client", this, out);
GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) {
gf_msg_debug (this->name, 0,
"fd lock list is empty");
fdctx->reopen_done (fdctx, fdctx->remote_fd, this);
} else {
lk_ctx = fdctx->lk_ctx;
LOCK (&lk_ctx->lock);
{
(void) _client_reacquire_lock (this, fdctx);
}
UNLOCK (&lk_ctx->lock);
}
ret = 0;
out:
return ret;
}
void
client_default_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
{
@ -670,7 +400,6 @@ client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
if (fd_count == 0) {
gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_UP_NOTIFY,
"last fd open'd/lock-self-heal'd - notifying CHILD-UP");
client_set_lk_version (this);
client_notify_parents_child_up (this);
}
}
@ -681,16 +410,13 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
{
int32_t ret = -1;
gfs3_open_rsp rsp = {0,};
gf_boolean_t attempt_lock_recovery = _gf_false;
clnt_local_t *local = NULL;
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
call_frame_t *frame = NULL;
xlator_t *this = NULL;
frame = myframe;
this = frame->this;
conf = this->private;
local = frame->local;
fdctx = local->fdctx;
@ -727,38 +453,10 @@ client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
goto out;
}
pthread_spin_lock (&conf->fd_lock);
{
if (!fdctx->released) {
if (conf->lk_heal &&
!client_fd_lk_list_empty (fdctx->lk_ctx,
_gf_false)) {
attempt_lock_recovery = _gf_true;
fdctx->lk_heal_state = GF_LK_HEAL_IN_PROGRESS;
}
}
}
pthread_spin_unlock (&conf->fd_lock);
ret = 0;
if (attempt_lock_recovery) {
/* Delay decrementing the reopen fd count until all the
locks corresponding to this fd are acquired.*/
gf_msg_debug (this->name, 0, "acquiring locks "
"on %s", local->loc.path);
ret = client_reacquire_lock (frame->this, local->fdctx);
if (ret) {
clnt_reacquire_lock_error (this, local->fdctx, conf);
gf_msg (this->name, GF_LOG_WARNING, 0,
PC_MSG_LOCK_ERROR, "acquiring locks failed "
"on %s", local->loc.path);
}
}
out:
if (!attempt_lock_recovery)
fdctx->reopen_done (fdctx, (rsp.op_ret) ? -1 : rsp.fd, this);
fdctx->reopen_done (fdctx, (rsp.op_ret) ? -1 : rsp.fd, this);
frame->local = NULL;
STACK_DESTROY (frame->root);
@ -1051,7 +749,6 @@ client_post_handshake (call_frame_t *frame, xlator_t *this)
gf_msg_debug (this->name, 0,
"No fds to open - notifying all parents child "
"up");
client_set_lk_version (this);
client_notify_parents_child_up (this);
}
out:
@ -1073,7 +770,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
int32_t op_ret = 0;
int32_t op_errno = 0;
gf_boolean_t auth_fail = _gf_false;
uint32_t lk_ver = 0;
glusterfs_ctx_t *ctx = NULL;
frame = myframe;
@ -1189,16 +885,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->child_up = (child_up_int != 0);
}
ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver);
if (ret) {
gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_GET_FAILED,
"failed to find key 'clnt-lk-version' in the options");
goto out;
}
gf_msg_debug (this->name, 0, "clnt-lk-version = %d, "
"server-lk-version = %d", client_get_lk_ver (conf),
lk_ver);
/* TODO: currently setpeer path is broken */
/*
if (process_uuid && req->conn &&
@ -1234,22 +920,7 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->connecting = 0;
conf->connected = 1;
if (lk_ver != client_get_lk_ver (conf)) {
gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_LOCK_MISMATCH,
"Server and Client lk-version numbers are not same, "
"reopening the fds");
client_mark_fd_bad (this);
client_post_handshake (frame, frame->this);
} else {
/*TODO: Traverse the saved fd list, and send
release to the server on fd's that were closed
during grace period */
gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_LOCK_MATCH,
"Server and Client lk-version numbers are same, no "
"need to reopen the fds");
client_notify_parents_child_up (frame->this);
}
client_post_handshake (frame, frame->this);
out:
if (auth_fail) {
gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_AUTH_FAILED,
@ -1334,21 +1005,16 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
}
}
/* When lock-heal is enabled:
* With multiple graphs possible in the same process, we need a
field to bring the uniqueness. Graph-ID should be enough to get the
job done.
* When lock-heal is disabled, connection-id should always be unique so
* that server never gets to reuse the previous connection resources
* so it cleans up the resources on every disconnect. Otherwise
* it may lead to stale resources, i.e. leaked file desciptors,
* inode/entry locks
*/
if (!conf->lk_heal) {
snprintf (counter_str, sizeof (counter_str),
/*
* Connection-id should always be unique so that server never gets to
* reuse the previous connection resources so it cleans up the resources
* on every disconnect. Otherwise it may lead to stale resources, i.e.
* leaked file desciptors, inode/entry locks
*/
snprintf (counter_str, sizeof (counter_str),
"-%"PRIu64, conf->setvol_count);
conf->setvol_count++;
}
conf->setvol_count++;
if (gethostname (hostname, 256) == -1) {
gf_msg (this->name, GF_LOG_ERROR, errno,
@ -1417,14 +1083,6 @@ client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
}
}
ret = dict_set_int16 (options, "clnt-lk-version",
client_get_lk_ver (conf));
if (ret < 0) {
gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAILED,
"failed to set clnt-lk-version(%"PRIu32") in handshake "
"msg", client_get_lk_ver (conf));
}
ret = dict_set_int32 (options, "opversion", GD_OP_VERSION_MAX);
if (ret < 0) {
gf_msg (this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED,
@ -1777,7 +1435,6 @@ char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
[GF_HNDSK_SETVOLUME] = "SETVOLUME",
[GF_HNDSK_GETSPEC] = "GETSPEC",
[GF_HNDSK_PING] = "PING",
[GF_HNDSK_SET_LK_VER] = "SET_LK_VER"
};
rpc_clnt_prog_t clnt_handshake_prog = {

19
xlators/protocol/client/src/client-rpc-fops.c
View File

@ -363,7 +363,6 @@ client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
fdctx->remote_fd = remote_fd;
fdctx->flags = flags;
fdctx->lk_ctx = fd_lk_ctx_ref (fd->lk_ctx);
fdctx->lk_heal_state = GF_LK_HEAL_DONE;
fdctx->reopen_done = client_default_reopen_done;
INIT_LIST_HEAD (&fdctx->sfd_pos);
@ -2437,20 +2436,6 @@ client3_3_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
goto out;
}
/* Save the lock to the client lock cache to be able
to recover in the case of server reboot.*/
/*
temporarily
if (local->cmd == F_SETLK || local->cmd == F_SETLKW) {
ret = client_add_lock_for_recovery (local->fd, &lock,
local->owner, local->cmd);
if (ret < 0) {
rsp.op_ret = -1;
rsp.op_errno = -ret;
}
}
*/
out:
if ((rsp.op_ret == -1) &&
(EAGAIN != gf_error_to_errno (rsp.op_errno))) {
@ -3277,7 +3262,6 @@ client3_3_release (call_frame_t *frame, xlator_t *this,
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
clnt_args_t *args = NULL;
lk_heal_state_t lk_heal_state = GF_LK_HEAL_DONE;
gf_boolean_t destroy = _gf_false;
if (!this || !data)
@ -3291,7 +3275,6 @@ client3_3_release (call_frame_t *frame, xlator_t *this,
pthread_spin_lock (&conf->fd_lock);
{
remote_fd = fdctx->remote_fd;
lk_heal_state = fdctx->lk_heal_state;
/* fdctx->remote_fd == -1 indicates a reopen attempt
in progress. Just mark ->released = 1 and let
@ -3299,7 +3282,7 @@ client3_3_release (call_frame_t *frame, xlator_t *this,
*/
if (remote_fd == -1) {
fdctx->released = 1;
} else if (lk_heal_state == GF_LK_HEAL_DONE) {
} else {
list_del_init (&fdctx->sfd_pos);
destroy = _gf_true;
}

18
xlators/protocol/client/src/client-rpc-fops_v2.c
View File

@ -2115,20 +2115,6 @@ client4_0_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
goto out;
}
/* Save the lock to the client lock cache to be able
to recover in the case of server reboot.*/
/*
temporarily
if (local->cmd == F_SETLK || local->cmd == F_SETLKW) {
ret = client_add_lock_for_recovery (local->fd, &lock,
local->owner, local->cmd);
if (ret < 0) {
rsp.op_ret = -1;
rsp.op_errno = -ret;
}
}
*/
out:
if ((rsp.op_ret == -1) &&
(EAGAIN != gf_error_to_errno (rsp.op_errno))) {
@ -2782,7 +2768,6 @@ client4_0_release (call_frame_t *frame, xlator_t *this,
clnt_conf_t *conf = NULL;
clnt_fd_ctx_t *fdctx = NULL;
clnt_args_t *args = NULL;
lk_heal_state_t lk_heal_state = GF_LK_HEAL_DONE;
gf_boolean_t destroy = _gf_false;
if (!this || !data)
@ -2796,7 +2781,6 @@ client4_0_release (call_frame_t *frame, xlator_t *this,
fdctx = this_fd_del_ctx (args->fd, this);
if (fdctx != NULL) {
remote_fd = fdctx->remote_fd;
lk_heal_state = fdctx->lk_heal_state;
/* fdctx->remote_fd == -1 indicates a reopen attempt
in progress. Just mark ->released = 1 and let
@ -2804,7 +2788,7 @@ client4_0_release (call_frame_t *frame, xlator_t *this,
*/
if (remote_fd == -1) {
fdctx->released = 1;
} else if (lk_heal_state == GF_LK_HEAL_DONE) {
} else {
list_del_init (&fdctx->sfd_pos);
destroy = _gf_true;
}

177
xlators/protocol/client/src/client.c
View File

@ -126,96 +126,6 @@ client_type_to_gf_type (short l_type)
return gf_type;
}
uint32_t
client_get_lk_ver (clnt_conf_t *conf)
{
uint32_t lk_ver = 0;
GF_VALIDATE_OR_GOTO ("client", conf, out);
pthread_mutex_lock (&conf->lock);
{
lk_ver = conf->lk_version;
}
pthread_mutex_unlock (&conf->lock);
out:
return lk_ver;
}
void
client_grace_timeout (void *data)
{
int ver = 0;
xlator_t *this = NULL;
struct clnt_conf *conf = NULL;
GF_VALIDATE_OR_GOTO ("client", data, out);
this = THIS;
conf = (struct clnt_conf *) this->private;
pthread_mutex_lock (&conf->lock);
{
ver = ++conf->lk_version;
/* ver == 0 is a special value used by server
to notify client that this is a fresh connect.*/
if (ver == 0)
ver = ++conf->lk_version;
gf_timer_call_cancel (this->ctx, conf->grace_timer);
conf->grace_timer = NULL;
}
pthread_mutex_unlock (&conf->lock);
gf_msg (this->name, GF_LOG_WARNING, 0, PC_MSG_TIMER_EXPIRED,
"client grace timer expired, updating "
"the lk-version to %d", ver);
client_mark_fd_bad (this);
out:
return;
}
int32_t
client_register_grace_timer (xlator_t *this, clnt_conf_t *conf)
{
int32_t ret = -1;
struct timespec grace_ts = {0, };
GF_VALIDATE_OR_GOTO ("client", this, out);
GF_VALIDATE_OR_GOTO (this->name, conf, out);
grace_ts.tv_sec = conf->grace_timeout;
grace_ts.tv_nsec = 0;
pthread_mutex_lock (&conf->lock);
{
if (conf->grace_timer || !conf->grace_timer_needed) {
gf_msg_trace (this->name, 0,
"Client grace timer is already set "
"or a grace-timer has already time "
"out, not registering a new timer");
} else {
gf_msg (this->name, GF_LOG_INFO, 0, PC_MSG_TIMER_REG,
"Registering a grace timer");
conf->grace_timer_needed = _gf_false;
conf->grace_timer =
gf_timer_call_after (this->ctx,
grace_ts,
client_grace_timeout,
conf->rpc);
}
}
pthread_mutex_unlock (&conf->lock);
ret = 0;
out:
return ret;
}
int
client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
@ -2298,34 +2208,12 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
gf_msg (this->name, GF_LOG_WARNING, 0,
PC_MSG_HANDSHAKE_RETURN, "handshake "
"msg returned %d", ret);
/* Cancel grace timer if set */
pthread_mutex_lock (&conf->lock);
{
conf->grace_timer_needed = _gf_true;
if (conf->grace_timer) {
gf_msg (this->name, GF_LOG_WARNING, 0,
PC_MSG_GRACE_TIMER_CANCELLED,
"Cancelling the grace timer");
gf_timer_call_cancel (this->ctx,
conf->grace_timer);
conf->grace_timer = NULL;
}
}
pthread_mutex_unlock (&conf->lock);
break;
}
case RPC_CLNT_DISCONNECT:
gf_msg_debug (this->name, 0, "got RPC_CLNT_DISCONNECT");
if (!conf->lk_heal)
client_mark_fd_bad (this);
else
client_register_grace_timer (this, conf);
client_mark_fd_bad (this);
if (!conf->skip_notify) {
if (conf->connected) {
@ -2378,9 +2266,7 @@ client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
} else {
rpc->conn.config.remote_port = 0;
}
break;
case RPC_CLNT_DESTROY:
ret = client_fini_complete (this);
break;
@ -2614,33 +2500,6 @@ out:
return ret;
}
int
client_init_grace_timer (xlator_t *this, dict_t *options,
clnt_conf_t *conf)
{
int32_t ret = -1;
GF_VALIDATE_OR_GOTO ("client", this, out);
GF_VALIDATE_OR_GOTO (this->name, options, out);
GF_VALIDATE_OR_GOTO (this->name, conf, out);
GF_OPTION_RECONF ("lk-heal", conf->lk_heal, options, bool, out);
gf_msg_debug (this->name, 0, "lk-heal = %s",
(conf->lk_heal) ? "on" : "off");
GF_OPTION_RECONF ("grace-timeout", conf->grace_timeout,
options, uint32, out);
gf_msg_debug (this->name, 0, "Client grace timeout value = %d",
conf->grace_timeout);
ret = 0;
out:
return ret;
}
int
client_check_event_threads (xlator_t *this, clnt_conf_t *conf, int32_t old,
int32_t new)
@ -2722,10 +2581,6 @@ reconfigure (xlator_t *this, dict_t *options)
GF_OPTION_RECONF ("send-gids", conf->send_gids, options, bool, out);
ret = client_init_grace_timer (this, options, conf);
if (ret)
goto out;
ret = 0;
out:
return ret;
@ -2761,11 +2616,6 @@ init (xlator_t *this)
conf->child_up = _gf_false;
/* Initialize parameters for lock self healing*/
conf->lk_version = 1;
conf->grace_timer = NULL;
conf->grace_timer_needed = _gf_true;
/* Set event threads to the configured default */
GF_OPTION_INIT("event-threads", conf->event_threads, int32, out);
ret = client_check_event_threads (this, conf, STARTING_EVENT_THREADS,
@ -2773,10 +2623,6 @@ init (xlator_t *this)
if (ret)
goto out;
ret = client_init_grace_timer (this, this->options, conf);
if (ret)
goto out;
LOCK_INIT (&conf->rec_lock);
conf->last_sent_event = -1; /* To start with we don't have any events */
@ -3078,27 +2924,6 @@ struct volume_options options[] = {
{ .key = {"client-bind-insecure"},
.type = GF_OPTION_TYPE_BOOL
},
{ .key = {"lk-heal"},
.type = GF_OPTION_TYPE_BOOL,
.default_value = "off",
.description = "When the connection to client is lost, server "
"cleans up all the locks held by the client. After "
"the connection is restored, the client reacquires "
"(heals) the fcntl locks released by the server.",
.op_version = {1},
.flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC
},
{ .key = {"grace-timeout"},
.type = GF_OPTION_TYPE_INT,
.min = 10,
.max = 1800,
.default_value = "10",
.description = "Specifies the duration for the lock state to be "
"maintained on the client after a network "
"disconnection. Range 10-1800 seconds.",
.op_version = {1},
.flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC
},
{ .key = {"tcp-window-size"},
.type = GF_OPTION_TYPE_SIZET,
.min = GF_MIN_SOCKET_WINDOW_SIZE,

24
xlators/protocol/client/src/client.h
View File

@ -33,11 +33,6 @@
#define GF_MAX_SOCKET_WINDOW_SIZE (1 * GF_UNIT_MB)
#define GF_MIN_SOCKET_WINDOW_SIZE (0)
typedef enum {
GF_LK_HEAL_IN_PROGRESS,
GF_LK_HEAL_DONE,
} lk_heal_state_t;
typedef enum {
DEFAULT_REMOTE_FD = 0,
FALLBACK_TO_ANON_FD = 1
@ -218,18 +213,6 @@ typedef struct clnt_conf {
char disconnect_err_logged; /* flag used to prevent
excessive disconnect
logging */
gf_boolean_t lk_heal;
uint16_t lk_version; /* this variable is used to distinguish
client-server transaction while
performing lock healing */
uint32_t grace_timeout;
gf_timer_t *grace_timer;
gf_boolean_t grace_timer_needed; /* The state of this flag will
be used to decide whether
a new grace-timer must be
registered or not. False
means dont register, true
means register */
char parent_down;
gf_boolean_t quick_reconnect; /* When reconnecting after
portmap query, do not let
@ -266,7 +249,6 @@ typedef struct _client_fd_ctx {
int32_t flags;
fd_lk_ctx_t *lk_ctx;
pthread_mutex_t mutex;
lk_heal_state_t lk_heal_state;
uuid_t gfid;
void (*reopen_done)(struct _client_fd_ctx*, int64_t rfd, xlator_t *);
struct list_head lock_list; /* List of all granted locks on this fd */
@ -376,8 +358,6 @@ int clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp);
int clnt_readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp);
int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx);
int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner);
int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock,
gf_lkowner_t *owner, int32_t cmd);
int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx);
int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd);
void client_save_number_fds (clnt_conf_t *conf, int count);
@ -388,14 +368,10 @@ int32_t client_dump_locks (char *name, inode_t *inode,
dict_t *dict);
int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
uint32_t client_get_lk_ver (clnt_conf_t *conf);
int32_t client_type_to_gf_type (short l_type);
int client_mark_fd_bad (xlator_t *this);
int client_set_lk_version (xlator_t *this);
int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
void client_default_reopen_done (clnt_fd_ctx_t *fdctx, int64_t rfd,
xlator_t *this);

81
xlators/protocol/server/src/server-handshake.c
View File

@ -465,9 +465,7 @@ server_setvolume (rpcsvc_request_t *req)
int32_t ret = -1;
int32_t op_ret = -1;
int32_t op_errno = EINVAL;
uint32_t lk_version = 0;
char *buf = NULL;
gf_boolean_t cancelled = _gf_false;
uint32_t opversion = 0;
rpc_transport_t *xprt = NULL;
int32_t fop_version = 0;
@ -617,20 +615,6 @@ server_setvolume (rpcsvc_request_t *req)
client_name = "unknown";
}
/*lk_verion :: [1..2^31-1]*/
ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version);
if (ret < 0) {
ret = dict_set_str (reply, "ERROR",
"lock state version not supplied");
if (ret < 0)
gf_msg_debug (this->name, 0, "failed to set error "
"msg");
op_ret = -1;
op_errno = EINVAL;
goto fail;
}
client = gf_client_get (this, &req->cred, client_uid, subdir_mount);
if (client == NULL) {
op_ret = -1;
@ -641,24 +625,6 @@ server_setvolume (rpcsvc_request_t *req)
client->client_name = gf_strdup(client_name);
gf_msg_debug (this->name, 0, "Connected to %s", client->client_uid);
cancelled = server_cancel_grace_timer (this, client);
if (cancelled) {
/* If timer has been successfully cancelled then it means
* that the client has reconnected within grace period.
* Since we've bumped up the bind count with a gf_client_get()
* for this connect attempt, we need to drop the bind count
* for earlier connect, since grace timer handler couldn't
* drop it since the timer was cancelled.
*/
gf_client_put (client, NULL);
/* We need to drop the ref count for this reconnected client
* since one ref was taken before delegating to the grace
* timer handler. Since grace timer handler was cancelled,
* it couldn't run and drop the ref either.
*/
gf_client_unref (client);
}
serv_ctx = server_ctx_get (client, client->this);
if (serv_ctx == NULL) {
@ -668,12 +634,6 @@ server_setvolume (rpcsvc_request_t *req)
goto fail;
}
if (serv_ctx->lk_version != 0 &&
serv_ctx->lk_version != lk_version) {
(void) server_connection_cleanup (this, client,
INTERNAL_LOCKS | POSIX_LOCKS);
}
if (req->trans->xl_private != client)
req->trans->xl_private = client;
@ -878,12 +838,6 @@ server_setvolume (rpcsvc_request_t *req)
if (ret)
gf_msg_debug (this->name, 0, "failed to set 'process-uuid'");
ret = dict_set_uint32 (reply, "clnt-lk-version", serv_ctx->lk_version);
if (ret)
gf_msg (this->name, GF_LOG_WARNING, 0,
PS_MSG_CLIENT_LK_VERSION_ERROR, "failed to set "
"'clnt-lk-version'");
ret = dict_set_uint64 (reply, "transport-ptr",
((uint64_t) (long) req->trans));
if (ret)
@ -976,47 +930,20 @@ server_ping (rpcsvc_request_t *req)
int
server_set_lk_version (rpcsvc_request_t *req)
{
int op_ret = -1;
int op_errno = EINVAL;
int ret = -1;
gf_set_lk_ver_req args = {0,};
gf_set_lk_ver_rsp rsp = {0,};
client_t *client = NULL;
server_ctx_t *serv_ctx = NULL;
xlator_t *this = NULL;
this = req->svc->xl;
//TODO: Decide on an appropriate errno for the error-path
//below
if (!this)
goto fail;
op_ret = xdr_to_generic (req->msg[0], &args,
ret = xdr_to_generic (req->msg[0], &args,
(xdrproc_t)xdr_gf_set_lk_ver_req);
if (op_ret < 0) {
//failed to decode msg;
if (ret < 0) {
/* failed to decode msg */
req->rpc_err = GARBAGE_ARGS;
goto fail;
}
client = gf_client_get (this, &req->cred, args.uid, NULL);
serv_ctx = server_ctx_get (client, client->this);
if (serv_ctx == NULL) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_SERVER_CTX_GET_FAILED, "server_ctx_get() "
"failed");
goto fail;
}
serv_ctx->lk_version = args.lk_ver;
rsp.lk_ver = args.lk_ver;
op_ret = 0;
fail:
if (client)
gf_client_put (client, NULL);
rsp.op_ret = op_ret;
rsp.op_errno = op_errno;
server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
(xdrproc_t)xdr_gf_set_lk_ver_rsp);

44
xlators/protocol/server/src/server-helpers.c
View File

@ -1277,49 +1277,6 @@ gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
return 0;
}
gf_boolean_t
server_cancel_grace_timer (xlator_t *this, client_t *client)
{
server_ctx_t *serv_ctx = NULL;
gf_timer_t *timer = NULL;
gf_boolean_t cancelled = _gf_false;
if (!this || !client) {
gf_msg (THIS->name, GF_LOG_ERROR, EINVAL, PS_MSG_INVALID_ENTRY,
"Invalid arguments to cancel connection timer");
return cancelled;
}
serv_ctx = server_ctx_get (client, client->this);
if (serv_ctx == NULL) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_SERVER_CTX_GET_FAILED,
"server_ctx_get() failed");
goto out;
}
LOCK (&serv_ctx->fdtable_lock);
{
if (serv_ctx->grace_timer) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_GRACE_TIMER_CANCELLED,
"Cancelling the grace timer");
timer = serv_ctx->grace_timer;
serv_ctx->grace_timer = NULL;
}
}
UNLOCK (&serv_ctx->fdtable_lock);
if (timer) {
gf_timer_call_cancel (this->ctx, timer);
cancelled = _gf_true;
}
out:
return cancelled;
}
server_ctx_t*
server_ctx_get (client_t *client, xlator_t *xlator)
{
@ -1339,7 +1296,6 @@ server_ctx_get (client_t *client, xlator_t *xlator)
if (ctx == NULL)
goto out;
/* ctx->lk_version = 0; redundant */
ctx->fdtable = gf_fd_fdtable_alloc ();
if (ctx->fdtable == NULL) {

3
xlators/protocol/server/src/server-helpers.h
View File

@ -44,9 +44,6 @@ int
server_connection_cleanup (xlator_t *this, struct _client *client,
int32_t flags);
gf_boolean_t
server_cancel_grace_timer (xlator_t *this, struct _client *client);
int
server_build_config (xlator_t *this, server_conf_t *conf);

200
xlators/protocol/server/src/server.c
View File

@ -31,61 +31,6 @@ rpcsvc_cbk_program_t server_cbk_prog = {
.progver = GLUSTER_CBK_VERSION,
};
void
grace_time_handler (void *data)
{
client_t *client = NULL;
xlator_t *this = NULL;
gf_timer_t *timer = NULL;
server_ctx_t *serv_ctx = NULL;
gf_boolean_t cancelled = _gf_false;
gf_boolean_t detached = _gf_false;
client = data;
this = client->this;
GF_VALIDATE_OR_GOTO (THIS->name, this, out);
gf_msg (this->name, GF_LOG_INFO, 0, PS_MSG_GRACE_TIMER_EXPD, "grace "
"timer expired for %s", client->client_uid);
serv_ctx = server_ctx_get (client, this);
if (serv_ctx == NULL) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_SERVER_CTX_GET_FAILED, "server_ctx_get() "
"failed");
goto out;
}
LOCK (&serv_ctx->fdtable_lock);
{
if (serv_ctx->grace_timer) {
timer = serv_ctx->grace_timer;
serv_ctx->grace_timer = NULL;
}
}
UNLOCK (&serv_ctx->fdtable_lock);
if (timer) {
gf_timer_call_cancel (this->ctx, timer);
cancelled = _gf_true;
}
if (cancelled) {
/*
* ref has already been taken in server_rpc_notify()
*/
gf_client_put (client, &detached);
if (client && detached) /* reconnection did not happen :-( */
server_connection_cleanup (this, client,
INTERNAL_LOCKS | POSIX_LOCKS);
gf_client_unref (client);
}
out:
return;
}
struct iobuf *
gfs_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
xdrproc_t xdrproc)
@ -145,7 +90,6 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
server_state_t *state = NULL;
char new_iobref = 0;
client_t *client = NULL;
gf_boolean_t lk_heal = _gf_false;
GF_VALIDATE_OR_GOTO ("server", req, ret);
@ -155,9 +99,6 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
client = frame->root->client;
}
if (client)
lk_heal = ((server_conf_t *) client->this->private)->lk_heal;
if (!iobref) {
iobref = iobref_new ();
if (!iobref) {
@ -193,7 +134,7 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
gf_msg_callingfn ("", GF_LOG_ERROR, 0,
PS_MSG_REPLY_SUBMIT_FAILED,
"Reply submission failed");
if (frame && client && !lk_heal) {
if (frame && client) {
server_connection_cleanup (frame->this, client,
INTERNAL_LOCKS | POSIX_LOCKS);
} else {
@ -472,8 +413,6 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
rpc_transport_t *trans = NULL;
server_conf_t *conf = NULL;
client_t *client = NULL;
server_ctx_t *serv_ctx = NULL;
struct timespec grace_ts = {0, };
char *auth_path = NULL;
int ret = -1;
@ -542,76 +481,24 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
auth_path = NULL;
}
/* If lock self heal is off, then destroy the
conn object, else register a grace timer event */
if (!conf->lk_heal) {
gf_client_ref (client);
gf_client_put (client, &detached);
if (client && detached) {
server_connection_cleanup (this, client,
INTERNAL_LOCKS | POSIX_LOCKS);
gf_event (EVENT_CLIENT_DISCONNECT,
"client_uid=%s;"
"client_identifier=%s;"
"server_identifier=%s;"
"brick_path=%s",
client->client_uid,
trans->peerinfo.identifier,
trans->myinfo.identifier,
auth_path);
}
/*
* gf_client_unref will be done while handling
* RPC_EVENT_TRANSPORT_DESTROY
*/
goto unref_transport;
gf_client_ref (client);
gf_client_put (client, &detached);
if (detached) {
server_connection_cleanup (this, client,
INTERNAL_LOCKS | POSIX_LOCKS);
gf_event (EVENT_CLIENT_DISCONNECT, "client_uid=%s;"
"client_identifier=%s;server_identifier=%s;"
"brick_path=%s",
client->client_uid,
trans->peerinfo.identifier,
trans->myinfo.identifier,
auth_path);
}
serv_ctx = server_ctx_get (client, this);
if (serv_ctx == NULL) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_SERVER_CTX_GET_FAILED,
"server_ctx_get() failed");
goto unref_transport;
}
grace_ts.tv_sec = conf->grace_timeout;
grace_ts.tv_nsec = 0;
LOCK (&serv_ctx->fdtable_lock);
{
if (!serv_ctx->grace_timer) {
gf_msg (this->name, GF_LOG_INFO, 0,
PS_MSG_GRACE_TIMER_START,
"starting a grace timer for %s",
client->client_uid);
/* ref to protect against client destruction
* in RPCSVC_EVENT_TRANSPORT_DESTROY while
* we are starting a grace timer
*/
gf_client_ref (client);
serv_ctx->grace_timer =
gf_timer_call_after (this->ctx,
grace_ts,
grace_time_handler,
client);
}
}
UNLOCK (&serv_ctx->fdtable_lock);
gf_event (EVENT_CLIENT_DISCONNECT, "client_uid=%s;"
"client_identifier=%s;server_identifier=%s;"
"brick_path=%s",
client->client_uid,
trans->peerinfo.identifier,
trans->myinfo.identifier,
auth_path);
/*
* gf_client_unref will be done while handling
* RPC_EVENT_TRANSPORT_DESTROY
*/
unref_transport:
/* rpc_transport_unref() causes a RPCSVC_EVENT_TRANSPORT_DESTROY
@ -619,17 +506,13 @@ unref_transport:
* So no code should ideally be after this unref
*/
rpc_transport_unref (trans);
break;
case RPCSVC_EVENT_TRANSPORT_DESTROY:
client = trans->xl_private;
if (!client)
break;
/* unref only for if (!client->lk_heal) */
if (!conf->lk_heal)
gf_client_unref (client);
gf_client_unref (client);
trans->xl_private = NULL;
break;
@ -703,33 +586,6 @@ _copy_auth_opt (dict_t *unused, char *key, data_t *value, void *xl_dict)
return 0;
}
int
server_init_grace_timer (xlator_t *this, dict_t *options,
server_conf_t *conf)
{
int32_t ret = -1;
GF_VALIDATE_OR_GOTO ("server", this, out);
GF_VALIDATE_OR_GOTO (this->name, options, out);
GF_VALIDATE_OR_GOTO (this->name, conf, out);
GF_OPTION_RECONF ("lk-heal", conf->lk_heal, options, bool, out);
gf_msg_debug (this->name, 0, "lk-heal = %s",
(conf->lk_heal) ? "on" : "off");
GF_OPTION_RECONF ("grace-timeout", conf->grace_timeout,
options, uint32, out);
gf_msg_debug (this->name, 0, "Server grace timeout value = %d",
conf->grace_timeout);
ret = 0;
out:
return ret;
}
int
server_check_event_threads (xlator_t *this, server_conf_t *conf, int32_t new)
{
@ -987,8 +843,6 @@ do_rpc:
if (ret)
goto out;
ret = server_init_grace_timer (this, options, conf);
out:
THIS = oldTHIS;
gf_msg_debug ("", 0, "returning %d", ret);
@ -1085,10 +939,6 @@ server_init (xlator_t *this)
if (ret)
goto out;
ret = server_init_grace_timer (this, this->options, conf);
if (ret)
goto out;
ret = server_build_config (this, conf);
if (ret)
goto out;
@ -1772,20 +1622,6 @@ struct volume_options server_options[] = {
.op_version = {1},
.flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC
},
{ .key = {"lk-heal"},
.type = GF_OPTION_TYPE_BOOL,
.default_value = "off",
.op_version = {1},
.flags = OPT_FLAG_SETTABLE
},
{.key = {"grace-timeout"},
.type = GF_OPTION_TYPE_INT,
.min = 10,
.max = 1800,
.default_value = "10",
.op_version = {1},
.flags = OPT_FLAG_SETTABLE
},
{.key = {"tcp-window-size"},
.type = GF_OPTION_TYPE_SIZET,
.min = GF_MIN_SOCKET_WINDOW_SIZE,

5
xlators/protocol/server/src/server.h
View File

@ -107,11 +107,8 @@ struct server_conf {
int inode_lru_limit;
gf_boolean_t verify_volfile;
gf_boolean_t trace;
gf_boolean_t lk_heal; /* If true means lock self
heal is on else off. */
char *conf_dir;
struct _volfile_ctx *volfile;
uint32_t grace_timeout;
dict_t *auth_modules;
pthread_mutex_t mutex;
struct list_head xprt_list;
@ -244,8 +241,6 @@ extern struct rpcsvc_program glusterfs4_0_fop_prog;
typedef struct _server_ctx {
gf_lock_t fdtable_lock;
fdtable_t *fdtable;
struct _gf_timer *grace_timer;
uint32_t lk_version;
} server_ctx_t;