andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

glusterd: volume-start, add-brick and remove-brick to use synctask framework

Browse Source 
- Added volume-id validation to glusterd-syncop code.
- All daemons are restarted using synctasks in init().
- glusterd_brick_start has wait/nowait variants to support
  volume commands using synctask framework and those that aren't.

Change-Id: Ieec26fe1ea7e5faac88cc7798d93e4cc2b399d34
BUG: 862834
Signed-off-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-on: http://review.gluster.org/3969
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
This commit is contained in:
Krishnan Parthasarathi 2012-10-11 22:43:17 +05:30 committed by Anand Avati
parent 1f2dbafc72
commit 6be13228c4
10 changed files with 193 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
xlators/mgmt/glusterd/src/glusterd-brick-ops.c
View File

@ -514,7 +514,7 @@ brick_val:
"failed to set the new type in dict");
}
ret = glusterd_op_begin (req, GD_OP_ADD_BRICK, dict);
ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
out:
if (ret) {
@ -531,9 +531,6 @@ out:
ret = 0; //sent error to cli, prevent second reply
}
glusterd_friend_sm ();
glusterd_op_sm ();
free (cli_req.dict.dict_val); //its malloced by xdr
return ret;
@ -806,7 +803,7 @@ glusterd_handle_remove_brick (rpcsvc_request_t *req)
}
}
ret = glusterd_op_begin (req, GD_OP_REMOVE_BRICK, dict);
ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
out:
if (ret) {
@ -828,9 +825,6 @@ out:
GF_FREE (brick_list);
free (cli_req.dict.dict_val); //its malloced by xdr
glusterd_friend_sm ();
glusterd_op_sm ();
return ret;
}
@ -938,7 +932,8 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
if (ret)
goto out;
ret = glusterd_brick_start (volinfo, brickinfo);
ret = glusterd_brick_start (volinfo, brickinfo,
_gf_true);
if (ret)
goto out;
i++;

4
xlators/mgmt/glusterd/src/glusterd-handler.c
View File

@ -2963,7 +2963,7 @@ struct rpcsvc_program gd_svc_mgmt_prog = {
.progver = GD_MGMT_VERSION,
.numactors = GLUSTERD_MGMT_MAXVALUE,
.actors = gd_svc_mgmt_actors,
.synctask = _gf_false,
.synctask = _gf_true,
};
rpcsvc_actor_t gd_svc_peer_actors[] = {
@ -3022,5 +3022,5 @@ struct rpcsvc_program gd_svc_cli_prog = {
.progver = GLUSTER_CLI_VERSION,
.numactors = GLUSTER_CLI_MAXVALUE,
.actors = gd_svc_cli_actors,
.synctask = _gf_false,
.synctask = _gf_true,
};

40
xlators/mgmt/glusterd/src/glusterd-op-sm.c
View File

@ -1022,7 +1022,7 @@ glusterd_start_bricks (glusterd_volinfo_t *volinfo)
glusterd_brickinfo_t *brickinfo = NULL;
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
if (glusterd_brick_start (volinfo, brickinfo))
if (glusterd_brick_start (volinfo, brickinfo, _gf_false))
return -1;
}
@ -1844,7 +1844,7 @@ out:
}
int
glusterd_op_build_payload (dict_t **req, char **op_errstr)
glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
{
int ret = -1;
void *ctx = NULL;
@ -1860,14 +1860,24 @@ glusterd_op_build_payload (dict_t **req, char **op_errstr)
if (!req_dict)
goto out;
op = glusterd_op_get_op ();
ctx = (void*)glusterd_op_get_ctx ();
if (!ctx) {
gf_log ("", GF_LOG_ERROR, "Null Context for "
"op %d", op);
ret = -1;
goto out;
if (!op_ctx) {
op = glusterd_op_get_op ();
ctx = (void*)glusterd_op_get_ctx ();
if (!ctx) {
gf_log ("", GF_LOG_ERROR, "Null Context for "
"op %d", op);
ret = -1;
goto out;
}
} else {
#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
if (ret)
goto out;
ctx = op_ctx;
}
#undef GD_SYNC_OPCODE_KEY
switch (op) {
case GD_OP_CREATE_VOLUME:
@ -2002,7 +2012,7 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
op = glusterd_op_get_op ();
ret = glusterd_op_build_payload (&dict, &op_errstr);
ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
if (ret) {
gf_log (THIS->name, GF_LOG_ERROR, "Building payload failed");
opinfo.op_errstr = op_errstr;
@ -2280,7 +2290,8 @@ out:
}
static int
glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx, glusterd_commit_hook_type_t type)
glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
glusterd_commit_hook_type_t type)
{
glusterd_conf_t *priv = NULL;
char hookdir[PATH_MAX] = {0, };
@ -2353,7 +2364,7 @@ glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
op = glusterd_op_get_op ();
op_dict = glusterd_op_get_ctx ();
ret = glusterd_op_build_payload (&dict, &op_errstr);
ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
if (ret) {
gf_log (THIS->name, GF_LOG_ERROR, "Building payload failed");
opinfo.op_errstr = op_errstr;
@ -2767,7 +2778,7 @@ glusterd_need_brick_op (glusterd_op_t op)
return ret;
}
static dict_t*
dict_t*
glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
{
dict_t *rsp_dict = NULL;
@ -4106,7 +4117,8 @@ glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
op = glusterd_op_get_op ();
req_ctx->op = op;
uuid_copy (req_ctx->uuid, MY_UUID);
ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr);
ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
NULL);
if (ret) {
gf_log (this->name, GF_LOG_ERROR,
"Building payload failed");

6
xlators/mgmt/glusterd/src/glusterd-op-sm.h
View File

@ -181,7 +181,7 @@ int32_t
glusterd_op_set_op (glusterd_op_t op);
int
glusterd_op_build_payload (dict_t **req, char **op_errstr);
glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
int32_t
glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
@ -242,6 +242,10 @@ int32_t
glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
gd_node_type type);
dict_t*
glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
int32_t
glusterd_op_init_ctx (glusterd_op_t op);
int32_t

5
xlators/mgmt/glusterd/src/glusterd-replace-brick.c
View File

@ -606,7 +606,8 @@ rb_src_brick_restart (glusterd_volinfo_t *volinfo,
}
sleep (2);
ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo);
ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
_gf_false);
if (ret) {
gf_log ("", GF_LOG_ERROR, "Unable to start "
"glusterfs, ret: %d", ret);
@ -1379,7 +1380,7 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t *volinfo,
goto out;
if (GLUSTERD_STATUS_STARTED == volinfo->status) {
ret = glusterd_brick_start (volinfo, new_brickinfo);
ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
if (ret)
goto out;
}

163
xlators/mgmt/glusterd/src/glusterd-syncop.c
View File

@ -310,6 +310,7 @@ out:
}
/*TODO: Need to add syncop for brick ops*/
int32_t
gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
int count, void *myframe)
@ -411,25 +412,80 @@ out:
}
static int
glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp,
char *op_errstr)
{
int ret = -1;
switch (op) {
case GD_OP_REPLACE_BRICK:
ret = glusterd_rb_use_rsp_dict (aggr, rsp);
if (ret)
goto out;
break;
case GD_OP_SYNC_VOLUME:
ret = glusterd_sync_use_rsp_dict (aggr, rsp);
if (ret)
goto out;
break;
case GD_OP_PROFILE_VOLUME:
ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
if (ret)
goto out;
break;
case GD_OP_GSYNC_SET:
ret = glusterd_gsync_use_rsp_dict (aggr, rsp, op_errstr);
if (ret)
goto out;
break;
case GD_OP_STATUS_VOLUME:
ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
if (ret)
goto out;
break;
case GD_OP_REBALANCE:
case GD_OP_DEFRAG_BRICK_VOLUME:
ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
if (ret)
goto out;
break;
case GD_OP_HEAL_VOLUME:
ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
if (ret)
goto out;
break;
default:
break;
}
out:
return ret;
}
int
gd_sync_task_begin (void *data)
gd_sync_task_begin (dict_t *op_ctx, char **op_errstr)
{
int ret = -1;
dict_t *dict = NULL;
dict_t *req_dict = NULL;
dict_t *rsp_dict = NULL;
glusterd_peerinfo_t *peerinfo = NULL;
glusterd_conf_t *conf = NULL;
uuid_t tmp_uuid = {0,};
char *errstr = NULL;
glusterd_op_t op = 0;
int32_t tmp_op = 0;
gf_boolean_t local_locked = _gf_false;
conf = THIS->private;
dict = data;
ret = dict_get_int32 (dict, GD_SYNC_OPCODE_KEY, &tmp_op);
ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
if (ret)
goto out;
@ -455,40 +511,60 @@ gd_sync_task_begin (void *data)
/* TODO: Only on lock successful nodes it should unlock */
}
ret = glusterd_op_build_payload (&req_dict, op_errstr, op_ctx);
if (ret)
goto out;
/* stage op */
ret = glusterd_op_stage_validate (op, dict, &errstr, rsp_dict);
ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
if (ret)
goto out;
list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
ret = gd_syncop_mgmt_stage_op (peerinfo->rpc,
conf->uuid, tmp_uuid,
op, dict, &rsp_dict, &errstr);
op, req_dict, &rsp_dict,
op_errstr);
if (ret) {
if (errstr)
ret = dict_set_dynstr (dict, "error", errstr);
if (*op_errstr)
ret = dict_set_dynstr (req_dict, "error",
*op_errstr);
ret = -1;
goto out;
}
if (op == GD_OP_REPLACE_BRICK)
(void) glusterd_syncop_aggr_rsp_dict (op, op_ctx,
rsp_dict,
*op_errstr);
if (rsp_dict)
dict_unref (rsp_dict);
}
/* commit op */
ret = glusterd_op_commit_perform (op, dict, &errstr, rsp_dict);
ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
if (ret)
goto out;
list_for_each_entry (peerinfo, &conf->peers, uuid_list) {
ret = gd_syncop_mgmt_commit_op (peerinfo->rpc,
conf->uuid, tmp_uuid,
op, dict, &rsp_dict, &errstr);
op, req_dict, &rsp_dict,
op_errstr);
if (ret) {
if (errstr)
ret = dict_set_dynstr (dict, "error", errstr);
if (*op_errstr)
ret = dict_set_dynstr (req_dict, "error",
*op_errstr);
ret = -1;
goto out;
}
(void) glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict,
*op_errstr);
if (rsp_dict)
dict_unref (rsp_dict);
}
ret = 0;
@ -509,57 +585,22 @@ out:
glusterd_unlock (conf->uuid);
}
if (req_dict)
dict_unref (req_dict);
if (rsp_dict)
dict_unref (rsp_dict);
return ret;
}
int
gd_sync_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
{
int ret = 0;
dict_t *dict = NULL;
rpcsvc_request_t *req = NULL;
int32_t tmp_op = 0;
glusterd_op_t op = 0;
dict = data;
req = sync_frame->local;
sync_frame->local = NULL;
ret = dict_get_int32 (dict, GD_SYNC_OPCODE_KEY, &tmp_op);
if (ret)
goto out;
op = tmp_op;
ret = glusterd_op_send_cli_response (op, op_ret, 0, req, NULL,
"operation failed");
out:
if (dict)
dict_unref (dict);
STACK_DESTROY (sync_frame->root);
return ret;
}
int32_t
glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
void *dict)
{
int ret = 0;
call_frame_t *dummy_frame = NULL;
glusterfs_ctx_t *ctx = NULL;
dummy_frame = create_frame (THIS, THIS->ctx->pool);
if (!dummy_frame)
goto out;
dummy_frame->local = req;
int op_ret = 0;
char *op_errstr = NULL;
ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
if (ret) {
@ -568,11 +609,15 @@ glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
goto out;
}
ctx = THIS->ctx;
ret = synctask_new (ctx->env, gd_sync_task_begin,
gd_sync_task_completion,
dummy_frame, dict);
op_ret = gd_sync_task_begin (dict, &op_errstr);
glusterd_op_send_cli_response (op, op_ret, 0, req, NULL,
op_errstr);
ret = 0;
out:
if (dict)
dict_unref (dict);
if (op_errstr)
GF_FREE (op_errstr);
return ret;
}

17
xlators/mgmt/glusterd/src/glusterd-utils.c
View File

@ -1155,7 +1155,8 @@ out:
int32_t
glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo)
glusterd_brickinfo_t *brickinfo,
gf_boolean_t wait)
{
int32_t ret = -1;
xlator_t *this = NULL;
@ -1306,7 +1307,11 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
runner_add_arg (&runner, "--mem-accounting");
runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
ret = runner_run_nowait (&runner);
if (wait)
ret = runner_run (&runner);
else
ret = runner_run_nowait (&runner);
if (ret)
goto out;
@ -3460,7 +3465,8 @@ out:
int
glusterd_brick_start (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo)
glusterd_brickinfo_t *brickinfo,
gf_boolean_t wait)
{
int ret = -1;
xlator_t *this = NULL;
@ -3488,7 +3494,7 @@ glusterd_brick_start (glusterd_volinfo_t *volinfo,
ret = 0;
goto out;
}
ret = glusterd_volume_start_glusterfs (volinfo, brickinfo);
ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
if (ret) {
gf_log ("", GF_LOG_ERROR, "Unable to start "
"glusterfs, ret: %d", ret);
@ -3513,7 +3519,8 @@ glusterd_restart_bricks (glusterd_conf_t *conf)
if (volinfo->status == GLUSTERD_STATUS_STARTED) {
list_for_each_entry (brickinfo, &volinfo->bricks,
brick_list) {
glusterd_brick_start (volinfo, brickinfo);
glusterd_brick_start (volinfo, brickinfo,
_gf_true);
}
start_nodesvcs = _gf_true;
}

10
xlators/mgmt/glusterd/src/glusterd-utils.h
View File

@ -124,7 +124,8 @@ glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
int32_t
glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo);
glusterd_brickinfo_t *brickinfo,
gf_boolean_t wait);
int32_t
glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo,
@ -242,8 +243,8 @@ int32_t
glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
dict_t *dict, int32_t count);
int
glusterd_get_brickinfo (xlator_t *this, const char *brickname,
int port, gf_boolean_t localhost,
glusterd_get_brickinfo (xlator_t *this, const char *brickname,
int port, gf_boolean_t localhost,
glusterd_brickinfo_t **brickinfo);
void
@ -267,7 +268,8 @@ glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
void *ctx);
int
glusterd_brick_start (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo);
glusterd_brickinfo_t *brickinfo,
gf_boolean_t wait);
int
glusterd_brick_stop (glusterd_volinfo_t *volinfo,
glusterd_brickinfo_t *brickinfo,

7
xlators/mgmt/glusterd/src/glusterd-volume-ops.c
View File

@ -260,14 +260,11 @@ glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
gf_log (this->name, GF_LOG_INFO, "Received start vol req"
" for volume %s", volname);
ret = glusterd_op_begin (req, GD_OP_START_VOLUME, dict);
ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
out:
free (cli_req.dict.dict_val); //its malloced by xdr
glusterd_friend_sm ();
glusterd_op_sm ();
if (ret) {
ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
dict, "operation failed");
@ -1415,7 +1412,7 @@ glusterd_op_start_volume (dict_t *dict, char **op_errstr)
if (ret)
goto out;
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
ret = glusterd_brick_start (volinfo, brickinfo);
ret = glusterd_brick_start (volinfo, brickinfo, _gf_true);
if (ret)
goto out;
}

34
xlators/mgmt/glusterd/src/glusterd.c
View File

@ -735,6 +735,29 @@ _install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
return -1;
}
static int
glusterd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
{
return ret;
}
static int
glusterd_launch_synctask (xlator_t *this, synctask_fn_t fn)
{
glusterd_conf_t *priv = NULL;
int ret = -1;
priv = this->private;
ret = synctask_new (this->ctx->env, fn,
glusterd_default_synctask_cbk, NULL, priv);
if (ret)
gf_log (this->name, GF_LOG_CRITICAL, "Failed to create synctask"
"for starting process");
return ret;
}
/*
* init - called during glusterd initialization
*
@ -1016,16 +1039,17 @@ init (xlator_t *this)
if (ret)
goto out;
glusterd_restart_bricks (conf);
ret = glusterd_restart_gsyncds (conf);
if (ret)
goto out;
glusterd_launch_synctask (this,
(synctask_fn_t) glusterd_restart_bricks);
glusterd_launch_synctask (this,
(synctask_fn_t) glusterd_restart_gsyncds);
glusterd_launch_synctask (this,
(synctask_fn_t) glusterd_restart_rebalance);
ret = glusterd_hooks_spawn_worker (this);
if (ret)
goto out;
glusterd_restart_rebalance (conf);
ret = 0;
out:
if (ret < 0) {