diff --git a/daemons/lvmlockd/lvmlockd-client.h b/daemons/lvmlockd/lvmlockd-client.h index 16d16132a..62ffb732a 100644 --- a/daemons/lvmlockd/lvmlockd-client.h +++ b/daemons/lvmlockd/lvmlockd-client.h @@ -14,6 +14,7 @@ #include "libdaemon/client/daemon-client.h" #define LVMLOCKD_SOCKET DEFAULT_RUN_DIR "/lvmlockd.socket" +#define LVMLOCKD_ADOPT_FILE DEFAULT_RUN_DIR "/lvmlockd.adopt" /* Wrappers to open/close connection */ diff --git a/daemons/lvmlockd/lvmlockd-core.c b/daemons/lvmlockd/lvmlockd-core.c index 39275fb17..84272c4d7 100644 --- a/daemons/lvmlockd/lvmlockd-core.c +++ b/daemons/lvmlockd/lvmlockd-core.c @@ -38,6 +38,8 @@ #define EXTERN #include "lvmlockd-internal.h" +static int str_to_mode(const char *str); + /* * Basic operation of lvmlockd * @@ -142,6 +144,8 @@ static const char *lvmlockd_protocol = "lvmlockd"; static const int lvmlockd_protocol_version = 1; static int daemon_quit; static int adopt_opt; +static uint32_t adopt_update_count; +static const char *adopt_file; /* * We use a separate socket for dumping daemon info. @@ -811,6 +815,144 @@ int version_from_args(char *args, unsigned int *major, unsigned int *minor, unsi return 0; } +/* + * Write new info when a command exits if that command has acquired a new LV + * lock. If the command has released an LV lock we don't bother updating the + * info. When adopting, we eliminate any LV lock adoptions if there is no dm + * device for that LV. If lvmlockd is terminated after acquiring but before + * writing this file, those LV locks would not be adopted on restart. + */ + +#define ADOPT_VERSION_MAJOR 1 +#define ADOPT_VERSION_MINOR 0 + +static void write_adopt_file(void) +{ + struct lockspace *ls; + struct resource *r; + struct lock *lk; + time_t t; + FILE *fp; + + if (!(fp = fopen(adopt_file, "w"))) + return; + + adopt_update_count++; + + t = time(NULL); + fprintf(fp, "lvmlockd adopt_version %u.%u pid %d updates %u %s", + ADOPT_VERSION_MAJOR, ADOPT_VERSION_MINOR, getpid(), adopt_update_count, ctime(&t)); + + pthread_mutex_lock(&lockspaces_mutex); + list_for_each_entry(ls, &lockspaces, list) { + if (ls->lm_type == LD_LM_DLM && !strcmp(ls->name, gl_lsname_dlm)) + continue; + fprintf(fp, "VG: %38s %s %s %s\n", + ls->vg_uuid, ls->vg_name, lm_str(ls->lm_type), ls->vg_args); + list_for_each_entry(r, &ls->resources, list) { + if (r->type != LD_RT_LV) + continue; + if ((r->mode != LD_LK_EX) && (r->mode != LD_LK_SH)) + continue; + list_for_each_entry(lk, &r->locks, list) { + fprintf(fp, "LV: %38s %s %s %s %u\n", + ls->vg_uuid, r->name, r->lv_args, mode_str(r->mode), r->version); + } + } + } + pthread_mutex_unlock(&lockspaces_mutex); + + fflush(fp); + fclose(fp); +} + +static int read_adopt_file(struct list_head *vg_lockd) +{ + char adopt_line[512]; + char vg_uuid[72]; + char lm_type_str[16]; + char mode[8]; + struct lockspace *ls, *ls2; + struct resource *r; + FILE *fp; + + if (MAX_ARGS != 64 || MAX_NAME != 64) + return -1; + + if (!(fp = fopen(adopt_file, "r"))) + return 0; + + while (fgets(adopt_line, sizeof(adopt_line), fp)) { + if (adopt_line[0] == '#') + continue; + else if (!strncmp(adopt_line, "lvmlockd", 8)) { + unsigned int v_major = 0, v_minor = 0; + sscanf(adopt_line, "lvmlockd adopt_version %u.%u", &v_major, &v_minor); + if (v_major != ADOPT_VERSION_MAJOR) + goto fail; + + } else if (!strncmp(adopt_line, "VG:", 3)) { + if (!(ls = alloc_lockspace())) + goto fail; + + memset(vg_uuid, 0, sizeof(vg_uuid)); + + if (sscanf(adopt_line, "VG: %63s %64s %16s %64s", + vg_uuid, ls->vg_name, lm_type_str, ls->vg_args) != 4) { + goto fail; + } + + memcpy(ls->vg_uuid, vg_uuid, 64); + + if ((ls->lm_type = str_to_lm(lm_type_str)) < 0) + goto fail; + + list_add(&ls->list, vg_lockd); + + } else if (!strncmp(adopt_line, "LV:", 3)) { + if (!(r = alloc_resource())) + goto fail; + + r->type = LD_RT_LV; + + memset(vg_uuid, 0, sizeof(vg_uuid)); + + if (sscanf(adopt_line, "LV: %64s %64s %s %8s %u", + vg_uuid, r->name, r->lv_args, mode, &r->version) != 5) { + goto fail; + } + + if ((r->adopt_mode = str_to_mode(mode)) == LD_LK_IV) + goto fail; + + if (ls && !memcmp(ls->vg_uuid, vg_uuid, 64)) { + list_add(&r->list, &ls->resources); + r = NULL; + } else { + list_for_each_entry(ls2, vg_lockd, list) { + if (memcmp(ls2->vg_uuid, vg_uuid, 64)) + continue; + list_add(&r->list, &ls2->resources); + r = NULL; + break; + } + } + + if (r) { + log_error("No lockspace found for resource %s vg_uuid %s", r->name, vg_uuid); + goto fail; + } + } + } + + fclose(fp); + return 0; + +fail: + fclose(fp); + return -1; +} + /* * These are few enough that arrays of function pointers can * be avoided. @@ -4689,6 +4831,7 @@ static void *client_thread_main(void *arg_in) struct client *cl; struct action *act; struct action *act_un; + uint32_t lock_acquire_count = 0, lock_acquire_written = 0; int rv; while (1) { @@ -4720,6 +4863,9 @@ static void *client_thread_main(void *arg_in) rv = -1; } + if (act->flags & LD_AF_LV_LOCK) + lock_acquire_count++; + /* * The client failed after we acquired an LV lock for * it, but before getting this reply saying it's done. @@ -4741,6 +4887,11 @@ static void *client_thread_main(void *arg_in) continue; } + if (adopt_opt && (lock_acquire_count > lock_acquire_written)) { + lock_acquire_written = lock_acquire_count; + write_adopt_file(); + } + /* * Queue incoming actions for lockspace threads */ @@ -4814,6 +4965,8 @@ static void *client_thread_main(void *arg_in) pthread_mutex_unlock(&client_mutex); } out: + if (adopt_opt && lock_acquire_written) + unlink(adopt_file); return NULL; } @@ -4846,180 +4999,6 @@ static void close_client_thread(void) log_error("pthread_join client_thread error %d", perrno); } -/* - * Get a list of all VGs with a lockd type (sanlock|dlm). - * We'll match this list against a list of existing lockspaces that are - * found in the lock manager. - * - * For each of these VGs, also create a struct resource on ls->resources to - * represent each LV in the VG that uses a lock. For each of these LVs - * that are active, we'll attempt to adopt a lock. - */ - -static int get_lockd_vgs(struct list_head *vg_lockd) -{ - /* FIXME: get VGs some other way */ - return -1; -#if 0 - struct list_head update_vgs; - daemon_reply reply; - struct dm_config_node *cn; - struct dm_config_node *metadata; - struct dm_config_node *md_cn; - struct dm_config_node *lv_cn; - struct lockspace *ls, *safe; - struct resource *r; - const char *vg_name; - const char *vg_uuid; - const char *lv_uuid; - const char *lock_type; - const char *lock_args; - char find_str_path[PATH_MAX]; - int rv = 0; - - INIT_LIST_HEAD(&update_vgs); - - reply = send_lvmetad("vg_list", "token = %s", "skip", NULL); - - if (reply.error || strcmp(daemon_reply_str(reply, "response", ""), "OK")) { - log_error("vg_list from lvmetad failed %d", reply.error); - rv = -EINVAL; - goto destroy; - } - - if (!(cn = dm_config_find_node(reply.cft->root, "volume_groups"))) { - log_error("get_lockd_vgs no vgs"); - rv = -EINVAL; - goto destroy; - } - - /* create an update_vgs list of all vg uuids */ - - for (cn = cn->child; cn; cn = cn->sib) { - vg_uuid = cn->key; - - if (!(ls = alloc_lockspace())) { - rv = -ENOMEM; - break; - } - - strncpy(ls->vg_uuid, vg_uuid, 64); - list_add_tail(&ls->list, &update_vgs); - log_debug("get_lockd_vgs %s", vg_uuid); - } - destroy: - daemon_reply_destroy(reply); - - if (rv < 0) - goto out; - - /* get vg_name and lock_type for each vg uuid entry in update_vgs */ - - list_for_each_entry(ls, &update_vgs, list) { - reply = send_lvmetad("vg_lookup", - "token = %s", "skip", - "uuid = %s", ls->vg_uuid, - NULL); - - if (reply.error || strcmp(daemon_reply_str(reply, "response", ""), "OK")) { - log_error("vg_lookup from lvmetad failed %d", reply.error); - rv = -EINVAL; - goto next; - } - - vg_name = daemon_reply_str(reply, "name", NULL); - if (!vg_name) { - log_error("get_lockd_vgs %s no name", ls->vg_uuid); - rv = -EINVAL; - goto next; - } - - strncpy(ls->vg_name, vg_name, MAX_NAME); - - metadata = dm_config_find_node(reply.cft->root, "metadata"); - if (!metadata) { - log_error("get_lockd_vgs %s name %s no metadata", - ls->vg_uuid, ls->vg_name); - rv = -EINVAL; - goto next; - } - - lock_type = dm_config_find_str(metadata, "metadata/lock_type", NULL); - ls->lm_type = str_to_lm(lock_type); - - if ((ls->lm_type != LD_LM_SANLOCK) && (ls->lm_type != LD_LM_DLM)) { - log_debug("get_lockd_vgs %s not lockd type", ls->vg_name); - continue; - } - - lock_args = dm_config_find_str(metadata, "metadata/lock_args", NULL); - if (lock_args) - strncpy(ls->vg_args, lock_args, MAX_ARGS); - - log_debug("get_lockd_vgs %s lock_type %s lock_args %s", - ls->vg_name, lock_type, lock_args ?: "none"); - - /* - * Make a record (struct resource) of each lv that uses a lock. - * For any lv that uses a lock, we'll check if the lv is active - * and if so try to adopt a lock for it. - */ - - for (md_cn = metadata->child; md_cn; md_cn = md_cn->sib) { - if (strcmp(md_cn->key, "logical_volumes")) - continue; - - for (lv_cn = md_cn->child; lv_cn; lv_cn = lv_cn->sib) { - snprintf(find_str_path, PATH_MAX, "%s/lock_args", lv_cn->key); - lock_args = dm_config_find_str(lv_cn, find_str_path, NULL); - if (!lock_args) - continue; - - snprintf(find_str_path, PATH_MAX, "%s/id", lv_cn->key); - lv_uuid = dm_config_find_str(lv_cn, find_str_path, NULL); - - if (!lv_uuid) { - log_error("get_lock_vgs no lv id for name %s", lv_cn->key); - continue; - } - - if (!(r = alloc_resource())) { - rv = -ENOMEM; - goto next; - } - - r->use_vb = 0; - r->type = LD_RT_LV; - strncpy(r->name, lv_uuid, MAX_NAME); - if (lock_args) - strncpy(r->lv_args, lock_args, MAX_ARGS); - list_add_tail(&r->list, &ls->resources); - log_debug("get_lockd_vgs %s lv %s %s (name %s)", - ls->vg_name, r->name, lock_args ? lock_args : "", lv_cn->key); - } - } - next: - daemon_reply_destroy(reply); - - if (rv < 0) - break; - } -out: - /* Return lockd VG's on the vg_lockd list. */ - - list_for_each_entry_safe(ls, safe, &update_vgs, list) { - list_del(&ls->list); - - if ((ls->lm_type == LD_LM_SANLOCK) || (ls->lm_type == LD_LM_DLM)) - list_add_tail(&ls->list, vg_lockd); - else - free(ls); - } - - return rv; -#endif -} - static char _dm_uuid[DM_UUID_LEN]; static char *get_dm_uuid(char *dm_name) @@ -5236,9 +5215,9 @@ static void adopt_locks(void) INIT_LIST_HEAD(&to_unlock); /* - * Get list of lockspaces from lock managers. - * Get list of VGs from lvmetad with a lockd type. - * Get list of active lockd type LVs from /dev. + * Get list of lockspaces from currently running lock managers. + * Get list of shared VGs from file written by prior lvmlockd. + * Get list of active LVs (in the shared VGs) from the file. */ if (lm_support_dlm() && lm_is_running_dlm()) { @@ -5262,12 +5241,17 @@ static void adopt_locks(void) * Adds a struct lockspace to vg_lockd for each lockd VG. * Adds a struct resource to ls->resources for each LV. */ - rv = get_lockd_vgs(&vg_lockd); + rv = read_adopt_file(&vg_lockd); if (rv < 0) { - log_error("adopt_locks get_lockd_vgs failed"); + log_error("adopt_locks read_adopt_file failed"); goto fail; } + if (list_empty(&vg_lockd)) { + log_debug("No lockspaces in adopt file"); + return; + } + /* * For each resource on each lockspace, check if the * corresponding LV is active. If so, leave the @@ -5506,7 +5490,7 @@ static void adopt_locks(void) goto fail; act->op = LD_OP_LOCK; act->rt = LD_RT_LV; - act->mode = LD_LK_EX; + act->mode = r->adopt_mode; act->flags = (LD_AF_ADOPT | LD_AF_PERSISTENT); act->client_id = INTERNAL_CLIENT_ID; act->lm_type = ls->lm_type; @@ -5604,8 +5588,9 @@ static void adopt_locks(void) * Adopt failed because the orphan has a different mode * than initially requested. Repeat the lock-adopt operation * with the other mode. N.B. this logic depends on first - * trying sh then ex for GL/VG locks, and ex then sh for - * LV locks. + * trying sh then ex for GL/VG locks; for LV locks the mode + * from the adopt file is tried first, the alternate + * (if the mode in adopt file was wrong somehow.) */ if ((act->rt != LD_RT_LV) && (act->mode == LD_LK_SH)) { @@ -5613,9 +5598,12 @@ static void adopt_locks(void) act->mode = LD_LK_EX; rv = add_lock_action(act); - } else if ((act->rt == LD_RT_LV) && (act->mode == LD_LK_EX)) { - /* LV locks: attempt to adopt sh after ex failed. */ - act->mode = LD_LK_SH; + } else if (act->rt == LD_RT_LV) { + /* LV locks: attempt to adopt the other mode. */ + if (act->mode == LD_LK_EX) + act->mode = LD_LK_SH; + else if (act->mode == LD_LK_SH) + act->mode = LD_LK_EX; rv = add_lock_action(act); } else { @@ -5750,10 +5738,13 @@ static void adopt_locks(void) if (count_start_fail || count_adopt_fail) goto fail; + unlink(adopt_file); + write_adopt_file(); log_debug("adopt_locks done"); return; fail: + unlink(adopt_file); log_error("adopt_locks failed, reset host"); } @@ -6028,6 +6019,8 @@ static void usage(char *prog, FILE *file) fprintf(file, " Set path to the pid file. [%s]\n", LVMLOCKD_PIDFILE); fprintf(file, " --socket-path | -s \n"); fprintf(file, " Set path to the socket to listen on. [%s]\n", LVMLOCKD_SOCKET); + fprintf(file, " --adopt-file \n"); + fprintf(file, " Set path to the adopt file. [%s]\n", LVMLOCKD_ADOPT_FILE); fprintf(file, " --syslog-priority | -S err|warning|debug\n"); fprintf(file, " Write log messages from this level up to syslog. [%s]\n", _syslog_num_to_name(LOG_SYSLOG_PRIO)); fprintf(file, " --gl-type | -g \n"); @@ -6063,6 +6056,7 @@ int main(int argc, char *argv[]) {"daemon-debug", no_argument, 0, 'D' }, {"pid-file", required_argument, 0, 'p' }, {"socket-path", required_argument, 0, 's' }, + {"adopt-file", required_argument, 0, 128 }, {"gl-type", required_argument, 0, 'g' }, {"host-id", required_argument, 0, 'i' }, {"host-id-file", required_argument, 0, 'F' }, @@ -6085,6 +6079,9 @@ int main(int argc, char *argv[]) switch (c) { case '0': break; + case 128: + adopt_file = strdup(optarg); + break; case 'h': usage(argv[0], stdout); exit(EXIT_SUCCESS); @@ -6146,6 +6143,9 @@ int main(int argc, char *argv[]) if (!ds.socket_path) ds.socket_path = LVMLOCKD_SOCKET; + if (!adopt_file) + adopt_file = LVMLOCKD_ADOPT_FILE; + /* runs daemon_main/main_loop */ daemon_start(ds); diff --git a/daemons/lvmlockd/lvmlockd-dlm.c b/daemons/lvmlockd/lvmlockd-dlm.c index 75e6deec4..7915cc008 100644 --- a/daemons/lvmlockd/lvmlockd-dlm.c +++ b/daemons/lvmlockd/lvmlockd-dlm.c @@ -398,12 +398,18 @@ static int lm_adopt_dlm(struct lockspace *ls, struct resource *r, int ld_mode, (void *)1, (void *)1, (void *)1, NULL, NULL); - if (rv == -1 && errno == -EAGAIN) { + if (rv == -1 && (errno == EAGAIN)) { log_debug("S %s R %s adopt_dlm adopt mode %d try other mode", ls->name, r->name, ld_mode); rv = -EUCLEAN; goto fail; } + if (rv == -1 && (errno == ENOENT)) { + log_debug("S %s R %s adopt_dlm adopt mode %d no lock", + ls->name, r->name, ld_mode); + rv = -ENOENT; + goto fail; + } if (rv < 0) { log_debug("S %s R %s adopt_dlm mode %d flags %x error %d errno %d", ls->name, r->name, mode, flags, rv, errno); diff --git a/daemons/lvmlockd/lvmlockd-internal.h b/daemons/lvmlockd/lvmlockd-internal.h index 85e8caf9a..04100c0c1 100644 --- a/daemons/lvmlockd/lvmlockd-internal.h +++ b/daemons/lvmlockd/lvmlockd-internal.h @@ -145,6 +145,7 @@ struct resource { char name[MAX_NAME+1]; /* vg name or lv name */ int8_t type; /* resource type LD_RT_ */ int8_t mode; + int8_t adopt_mode; unsigned int sh_count; /* number of sh locks on locks list */ uint32_t version; uint32_t last_client_id; /* last client_id to lock or unlock resource */ diff --git a/man/lvmlockd.8_main b/man/lvmlockd.8_main index 8ed5400e4..c21f7a916 100644 --- a/man/lvmlockd.8_main +++ b/man/lvmlockd.8_main @@ -58,6 +58,10 @@ For default settings, see lvmlockd -h. .I path Set path to the socket to listen on. +.B --adopt-file +.I path + Set path to the adopt file. + .B --syslog-priority | -S err|warning|debug Write log messages from this level up to syslog. @@ -76,6 +80,8 @@ For default settings, see lvmlockd -h. .I seconds Override the default sanlock I/O timeout. +.B --adopt | -A 0|1 + Enable (1) or disable (0) lock adoption. .SH USAGE @@ -548,7 +554,13 @@ necessary locks. .B lvmlockd failure If lvmlockd fails or is killed while holding locks, the locks are orphaned -in the lock manager. +in the lock manager. Orphaned locks must be cleared or adopted before the +associated resources can be accessed normally. If lock adoption is +enabled, lvmlockd keeps a record of locks in the adopt-file. A subsequent +instance of lvmlockd will then adopt locks orphaned by the previous +instance. Adoption must be enabled in both instances (--adopt|-A 1). +Without adoption, the lock manager or host would require a reset to clear +orphaned lock state. .B dlm/corosync failure