glusterd: initialize the daemon services on demand

As of now all the daemon services are initialized at glusterD init path. Since
socket file path of per node daemon demands the uuid of the node, MY_UUID macro
is invoked as part of the initialization.

The above flow breaks the usecases where a gluster image is built following a
template could be Dockerfile, Vagrantfile or any kind of virtualization
environment. This means bringing instances of this image would have same UUIDs
for the node resulting in peer probe failure.

Solution is to lazily initialize the services on demand.

Change-Id: If7caa533026c83e98c7c7678bded67085d0bbc1e
BUG: 1238135
Signed-off-by: Atin Mukherjee <amukherj@redhat.com>
Reviewed-on: http://review.gluster.org/11488
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Gaurav Kumar Garg <ggarg@redhat.com>
Reviewed-by: Kaushal M <kaushal@redhat.com>
This commit is contained in:
Atin Mukherjee 2015-07-01 14:47:48 +05:30 committed by Kaushal M
parent fd296d9016
commit d49b4b1086
23 changed files with 214 additions and 150 deletions

View File

@ -25,9 +25,12 @@ Data members & functions of different management objects
- connection object
- process object
- online status
- Methods - manager, start, stop which can be abstracted as a common methods
or specific to service requirements
- init API can be invoked using the service management object
- Methods
-- manager, start, stop which can be abstracted as a common
methods or specific to service requirements
-- init API is invoked on demand of the service and currently integrated
into manager.
-- build method is to initialize the method pointers
The above structures defines the skeleton of the daemon management framework.
Introduction of new daemons in GlusterFS needs to inherit these properties. Any

View File

@ -12,7 +12,9 @@ TEST launch_cluster 2;
# is > 3.5. This is to ensure 3.5 code path is hit to test that volume status
# works when a node is upgraded from 3.5 to 3.7 or higher as mgmt_v3 lock is
# been introduced in 3.6 version and onwards
GD1_WD=$($CLI_1 system getwd)
$CLI_1 system uuid get
TEST sed -rnie "'s/(operating-version=)\w+/\130500/gip'" ${GD1_WD}/glusterd.info
TEST kill_glusterd 1

View File

@ -0,0 +1,16 @@
#!/bin/bash
. $(dirname $0)/../../include.rc
cleanup;
TEST glusterd;
TEST pidof glusterd;
GDWD=$($CLI system getwd)
# glusterd.info file will be created on either first peer probe or volume
# creation, hence we expect file to be not present in this case
TEST ! -e $GDWD/glusterd.info
cleanup;

View File

@ -15,13 +15,18 @@
#include "glusterd-volgen.h"
#include "glusterd-bitd-svc.h"
void
glusterd_bitdsvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_bitdsvc_manager;
svc->start = glusterd_bitdsvc_start;
svc->stop = glusterd_bitdsvc_stop;
}
int
glusterd_bitdsvc_init (glusterd_svc_t *svc)
{
return glusterd_svc_init (svc, bitd_svc_name,
glusterd_bitdsvc_manager,
glusterd_bitdsvc_start,
glusterd_bitdsvc_stop);
return glusterd_svc_init (svc, bitd_svc_name);
}
static int
@ -65,6 +70,20 @@ glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
this = THIS;
GF_ASSERT (this);
if (!svc->inited) {
ret = glusterd_bitdsvc_init (svc);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_BITD_INIT_FAIL, "Failed to init "
"bitd service");
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (this->name, 0, "BitD service "
"initialized");
}
}
if (glusterd_should_i_stop_bitd ()) {
ret = svc->stop (svc, SIGTERM);
} else {

View File

@ -15,6 +15,9 @@
#define bitd_svc_name "bitd"
void
glusterd_bitdsvc_build (glusterd_svc_t *svc);
int
glusterd_bitdsvc_init (glusterd_svc_t *svc);

View File

@ -666,14 +666,19 @@ start_ganesha (char **op_errstr)
}
}
ret = priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
if (ret) {
ret = -1;
gf_asprintf (op_errstr, "Gluster-NFS service could"
"not be stopped, exiting.");
goto out;
/* If the nfs svc is not initialized it means that the service is not
* running, hence we can skip the process of stopping gluster-nfs
* service
*/
if (priv->nfs_svc.inited) {
ret = priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
if (ret) {
ret = -1;
gf_asprintf (op_errstr, "Gluster-NFS service could"
"not be stopped, exiting.");
goto out;
}
}
if (check_host_list()) {
ret = manage_service ("start");
if (ret)

View File

@ -18,6 +18,14 @@
char *nfs_svc_name = "nfs";
void
glusterd_nfssvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_nfssvc_manager;
svc->start = glusterd_nfssvc_start;
svc->stop = glusterd_nfssvc_stop;
}
static gf_boolean_t
glusterd_nfssvc_need_start ()
{
@ -43,10 +51,7 @@ glusterd_nfssvc_need_start ()
int
glusterd_nfssvc_init (glusterd_svc_t *svc)
{
return glusterd_svc_init (svc, nfs_svc_name,
glusterd_nfssvc_manager,
glusterd_nfssvc_start,
glusterd_nfssvc_stop);
return glusterd_svc_init (svc, nfs_svc_name);
}
static int
@ -167,6 +172,19 @@ glusterd_nfssvc_manager (glusterd_svc_t *svc, void *data, int flags)
{
int ret = -1;
if (!svc->inited) {
ret = glusterd_nfssvc_init (svc);
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_NFSSVC, "Failed to init nfs "
"service");
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (THIS->name, 0, "nfs service initialized");
}
}
ret = svc->stop (svc, SIGKILL);
if (ret)
goto out;

View File

@ -13,6 +13,9 @@
#include "glusterd-svc-mgmt.h"
void
glusterd_nfssvc_build (glusterd_svc_t *svc);
int
glusterd_nfssvc_init (glusterd_svc_t *svc);

View File

@ -18,16 +18,21 @@
char *quotad_svc_name = "quotad";
void
glusterd_quotadsvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_quotadsvc_manager;
svc->start = glusterd_quotadsvc_start;
svc->stop = glusterd_svc_stop;
}
int glusterd_quotadsvc_init (glusterd_svc_t *svc)
{
int ret = -1;
char volfile[PATH_MAX] = {0,};
glusterd_conf_t *conf = THIS->private;
ret = glusterd_svc_init (svc, quotad_svc_name,
glusterd_quotadsvc_manager,
glusterd_quotadsvc_start,
glusterd_svc_stop);
ret = glusterd_svc_init (svc, quotad_svc_name);
if (ret)
goto out;
@ -60,6 +65,20 @@ glusterd_quotadsvc_manager (glusterd_svc_t *svc, void *data, int flags)
int ret = 0;
glusterd_volinfo_t *volinfo = NULL;
if (!svc->inited) {
ret = glusterd_quotadsvc_init (svc);
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_QUOTASVC, "Failed to init "
"quotad service");
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (THIS->name, 0, "quotad service "
"initialized");
}
}
volinfo = data;
/* If all the volumes are stopped or all shd compatible volumes

View File

@ -13,6 +13,9 @@
#include "glusterd-svc-mgmt.h"
void
glusterd_quotadsvc_build (glusterd_svc_t *svc);
int
glusterd_quotadsvc_init (glusterd_svc_t *svc);

View File

@ -17,13 +17,18 @@
char *scrub_svc_name = "scrub";
void
glusterd_scrubsvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_scrubsvc_manager;
svc->start = glusterd_scrubsvc_start;
svc->stop = glusterd_scrubsvc_stop;
}
int
glusterd_scrubsvc_init (glusterd_svc_t *svc)
{
return glusterd_svc_init (svc, scrub_svc_name,
glusterd_scrubsvc_manager,
glusterd_scrubsvc_start,
glusterd_scrubsvc_stop);
return glusterd_svc_init (svc, scrub_svc_name);
}
static int
@ -60,6 +65,20 @@ glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags)
{
int ret = -EINVAL;
if (!svc->inited) {
ret = glusterd_scrubsvc_init (svc);
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_SCRUB_INIT_FAIL, "Failed to init "
"scrub service");
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (THIS->name, 0, "scrub service "
"initialized");
}
}
if (glusterd_should_i_stop_bitd ()) {
ret = svc->stop (svc, SIGTERM);
} else {

View File

@ -20,6 +20,9 @@ struct glusterd_scrubsvc_{
gf_store_handle_t *handle;
};
void
glusterd_scrubsvc_build (glusterd_svc_t *svc);
int
glusterd_scrubsvc_init (glusterd_svc_t *svc);

View File

@ -18,13 +18,18 @@
char *shd_svc_name = "glustershd";
void
glusterd_shdsvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_shdsvc_manager;
svc->start = glusterd_shdsvc_start;
svc->stop = glusterd_svc_stop;
}
int
glusterd_shdsvc_init (glusterd_svc_t *svc)
{
return glusterd_svc_init (svc, shd_svc_name,
glusterd_shdsvc_manager,
glusterd_shdsvc_start,
glusterd_svc_stop);
return glusterd_svc_init (svc, shd_svc_name);
}
static int
@ -80,6 +85,19 @@ glusterd_shdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
int ret = 0;
glusterd_volinfo_t *volinfo = NULL;
if (!svc->inited) {
ret = glusterd_shdsvc_init (svc);
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_SHDSVC, "Failed to init shd "
"service");
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (THIS->name, 0, "shd service initialized");
}
}
volinfo = data;
/* If all the volumes are stopped or all shd compatible volumes

View File

@ -13,6 +13,9 @@
#include "glusterd-svc-mgmt.h"
void
glusterd_shdsvc_build (glusterd_svc_t *svc);
int
glusterd_shdsvc_init (glusterd_svc_t *svc);

View File

@ -36,6 +36,13 @@ glusterd_svc_build_snapd_logfile (char *logfile, char *logdir, size_t len)
snprintf (logfile, len, "%s/snapd.log", logdir);
}
void
glusterd_snapdsvc_build (glusterd_svc_t *svc)
{
svc->manager = glusterd_snapdsvc_manager;
svc->start = glusterd_snapdsvc_start;
svc->stop = glusterd_svc_stop;
}
int
glusterd_snapdsvc_init (void *data)
@ -69,10 +76,6 @@ glusterd_snapdsvc_init (void *data)
if (ret < 0)
goto out;
svc->manager = glusterd_snapdsvc_manager;
svc->start = glusterd_snapdsvc_start;
svc->stop = glusterd_svc_stop;
notify = glusterd_snapdsvc_rpc_notify;
glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
@ -123,6 +126,21 @@ glusterd_snapdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
volinfo = data;
if (!svc->inited) {
ret = glusterd_snapdsvc_init (volinfo);
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize "
"snapd service for volume %s",
volinfo->volname);
goto out;
} else {
svc->inited = _gf_true;
gf_msg_debug (THIS->name, 0, "snapd service "
"initialized");
}
}
ret = glusterd_is_snapd_enabled (volinfo);
if (ret == -1) {
gf_msg (this->name, GF_LOG_ERROR, 0,

View File

@ -21,6 +21,9 @@ struct glusterd_snapdsvc_{
gf_store_handle_t *handle;
};
void
glusterd_snapdsvc_build (glusterd_svc_t *svc);
int
glusterd_snapdsvc_init (void *data);

View File

@ -6488,7 +6488,6 @@ glusterd_snapshot_clone_commit (dict_t *dict, char **op_errstr,
snap->snapname);
goto out;
}
ret = glusterd_snapdsvc_init (snap_vol);
glusterd_list_add_order (&snap_vol->vol_list, &priv->volumes,
glusterd_compare_volume_name);
@ -9521,15 +9520,6 @@ gd_restore_snap_volume (dict_t *dict, dict_t *rsp_dict,
goto out;
}
/* Initialize the snapd service */
ret = glusterd_snapdsvc_init (new_volinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize snapd "
"service for volume %s", orig_vol->volname);
goto out;
}
ret = 0;
out:
if (ret) {

View File

@ -2810,15 +2810,6 @@ glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
if (snap)
volinfo->is_snap_volume = _gf_true;
/* Initialize the snapd service */
ret = glusterd_snapdsvc_init (volinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize snapd "
"service for volume %s", volinfo->volname);
goto out;
}
ret = glusterd_store_update_volinfo (volinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,

View File

@ -49,9 +49,6 @@ static int
glusterd_svc_init_common (glusterd_svc_t *svc,
char *svc_name, char *workdir,
char *rundir, char *logdir,
glusterd_svc_manager_t manager,
glusterd_svc_start_t start,
glusterd_svc_stop_t stop,
glusterd_conn_notify_t notify)
{
int ret = -1;
@ -74,10 +71,6 @@ glusterd_svc_init_common (glusterd_svc_t *svc,
if (ret < 0)
goto out;
svc->manager = manager;
svc->start = start;
svc->stop = stop;
if (!notify)
notify = glusterd_svc_common_rpc_notify;
@ -126,10 +119,7 @@ svc_add_args (dict_t *cmdline, char *arg, data_t *value, void *data)
return 0;
}
int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name,
glusterd_svc_manager_t manager,
glusterd_svc_start_t start,
glusterd_svc_stop_t stop)
int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name)
{
int ret = -1;
char rundir[PATH_MAX] = {0,};
@ -145,8 +135,7 @@ int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name,
glusterd_svc_build_rundir (svc_name, priv->workdir, rundir,
sizeof (rundir));
ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir,
DEFAULT_LOG_FILE_DIRECTORY, manager,
start, stop, NULL);
DEFAULT_LOG_FILE_DIRECTORY, NULL);
return ret;
}

View File

@ -17,6 +17,8 @@
struct glusterd_svc_;
typedef struct glusterd_svc_ glusterd_svc_t;
typedef void (*glusterd_svc_build_t) (glusterd_svc_t *svc);
typedef int (*glusterd_svc_manager_t) (glusterd_svc_t *svc,
void *data, int flags);
typedef int (*glusterd_svc_start_t) (glusterd_svc_t *svc, int flags);
@ -26,20 +28,19 @@ struct glusterd_svc_ {
char name[PATH_MAX];
glusterd_conn_t conn;
glusterd_proc_t proc;
glusterd_svc_build_t build;
glusterd_svc_manager_t manager;
glusterd_svc_start_t start;
glusterd_svc_stop_t stop;
gf_boolean_t online;
gf_boolean_t inited;
};
int
glusterd_svc_create_rundir (char *rundir);
int
glusterd_svc_init (glusterd_svc_t *svc, char *svc_name,
glusterd_svc_manager_t manager,
glusterd_svc_start_t start,
glusterd_svc_stop_t stop);
glusterd_svc_init (glusterd_svc_t *svc, char *svc_name);
int
glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline);

View File

@ -496,6 +496,9 @@ glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
new_volinfo->xl = THIS;
new_volinfo->snapd.svc.build = glusterd_snapdsvc_build;
new_volinfo->snapd.svc.build (&(new_volinfo->snapd.svc));
pthread_mutex_init (&new_volinfo->reflock, NULL);
*volinfo = glusterd_volinfo_ref (new_volinfo);
@ -3772,35 +3775,17 @@ glusterd_import_friend_volume (dict_t *peer_data, size_t count)
ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
if (0 == ret) {
/* snapdsvc initialization of old_volinfo is also required here
* as glusterd_delete_stale_volume () invokes snapdsvc manager
*/
ret = glusterd_snapdsvc_init (old_volinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize"
" snapdsvc for old volume %s",
old_volinfo->volname);
goto out;
}
(void) gd_check_and_update_rebalance_info (old_volinfo,
new_volinfo);
(void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
}
ret = glusterd_snapdsvc_init (new_volinfo);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize "
"snapdsvc for volume %s", new_volinfo->volname);
goto out;
}
if (glusterd_is_volume_started (new_volinfo)) {
(void) glusterd_start_bricks (new_volinfo);
if (glusterd_is_snapd_enabled (new_volinfo)) {
svc = &(new_volinfo->snapd.svc);
(void) svc->start (svc, PROC_START_NO_WAIT);
(void) svc->manager (svc, new_volinfo,
PROC_START_NO_WAIT);
}
}

View File

@ -2297,12 +2297,6 @@ glusterd_op_create_volume (dict_t *dict, char **op_errstr)
goto out;
}
ret = glusterd_snapdsvc_init (volinfo);
if (ret) {
*op_errstr = gf_strdup ("Failed to initialize snapd service");
goto out;
}
ret = glusterd_create_volfiles_and_notify_services (volinfo);
if (ret) {
*op_errstr = gf_strdup ("Failed to create volume files");

View File

@ -1271,10 +1271,9 @@ out:
return ret;
}
static int
glusterd_svc_init_all ()
static void
glusterd_svcs_build ()
{
int ret = -1;
xlator_t *this = NULL;
glusterd_conf_t *priv = NULL;
@ -1284,59 +1283,22 @@ glusterd_svc_init_all ()
priv = this->private;
GF_ASSERT (priv);
/* Init SHD svc */
ret = glusterd_shdsvc_init (&(priv->shd_svc));
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_SHDSVC,
"Failed to init shd service");
goto out;
}
gf_msg_debug (THIS->name, 0, "shd service initialized");
priv->shd_svc.build = glusterd_shdsvc_build;
priv->shd_svc.build (&(priv->shd_svc));
/* Init NFS svc */
ret = glusterd_nfssvc_init (&(priv->nfs_svc));
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_NFSSVC,
"Failed to init nfs service");
goto out;
}
gf_msg_debug (THIS->name, 0, "nfs service initialized");
priv->nfs_svc.build = glusterd_nfssvc_build;
priv->nfs_svc.build (&(priv->nfs_svc));
/* Init QuotaD svc */
ret = glusterd_quotadsvc_init (&(priv->quotad_svc));
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_FAILED_INIT_QUOTASVC, "Failed to init quotad "
"service");
goto out;
}
gf_msg_debug (THIS->name, 0, "quotad service initialized");
priv->quotad_svc.build = glusterd_quotadsvc_build;
priv->quotad_svc.build (&(priv->quotad_svc));
/* Init BitD svc */
ret = glusterd_bitdsvc_init (&(priv->bitd_svc));
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_BITD_INIT_FAIL, "Failed to initialized BitD "
"service");
goto out;
}
gf_msg_debug (THIS->name, 0, "BitD service initialized");
priv->bitd_svc.build = glusterd_bitdsvc_build;
priv->bitd_svc.build (&(priv->bitd_svc));
ret = glusterd_scrubsvc_init (&(priv->scrub_svc));
if (ret) {
gf_msg (THIS->name, GF_LOG_ERROR, 0,
GD_MSG_SCRUB_INIT_FAIL, "Failed to initialized scrub "
"service");
goto out;
}
gf_msg_debug (THIS->name, 0, "scrub service initialized");
priv->scrub_svc.build = glusterd_scrubsvc_build;
priv->scrub_svc.build (&(priv->scrub_svc));
out:
return ret;
}
/*
* init - called during glusterd initialization
*
@ -1738,6 +1700,7 @@ init (xlator_t *this)
this->private = conf;
glusterd_mgmt_v3_lock_init ();
glusterd_txn_opinfo_dict_init ();
glusterd_svcs_build ();
/* Make install copies few of the hook-scripts by creating hooks
* directory. Hence purposefully not doing the check for the presence of
@ -1787,10 +1750,6 @@ init (xlator_t *this)
goto out;
}
ret = glusterd_svc_init_all ();
if (ret)
goto out;
ret = glusterd_restore ();
if (ret < 0)
goto out;