From c23e7ff2a02c65619d05eb525adaaf9336de2692 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Fri, 7 Nov 2014 12:51:42 -0600 Subject: [PATCH] Reread global state the lvmetad copy is stale This is the client side handling of the global_invalid state added to lvmetad in commit c595b50cec8a6b95c6ac4988912d1412f3cc0237. The function added here: . checks if the global state in lvmetad is invalid . if so, scans disks to update the state in lvmetad . clears the global_invalid flag in lvmetad . updates the local udev db to reflect any changes --- lib/cache/lvmetad.c | 296 ++++++++++++++++++++++++++++++++++++++++++++ lib/cache/lvmetad.h | 2 + 2 files changed, 298 insertions(+) diff --git a/lib/cache/lvmetad.c b/lib/cache/lvmetad.c index 3c07cd1ea..01c3f08a0 100644 --- a/lib/cache/lvmetad.c +++ b/lib/cache/lvmetad.c @@ -1283,3 +1283,299 @@ int lvmetad_vg_clear_outdated_pvs(struct volume_group *vg) return result; } + +/* + * Records the state of cached PVs in lvmetad so we can look for changes + * after rescanning. + */ +struct pv_cache_list { + struct dm_list list; + dev_t devt; + struct id pvid; + const char *vgid; + unsigned found : 1; + unsigned update_udev : 1; +}; + +/* + * Get the list of PVs known to lvmetad. + */ +static int _lvmetad_get_pv_cache_list(struct cmd_context *cmd, struct dm_list *pvc_list) +{ + daemon_reply reply; + struct dm_config_node *cn; + struct pv_cache_list *pvcl; + const char *pvid_txt; + const char *vgid; + + if (!lvmetad_active()) + return 1; + + log_debug_lvmetad("Asking lvmetad for complete list of known PVs"); + reply = _lvmetad_send("pv_list", NULL); + if (!_lvmetad_handle_reply(reply, "list PVs", "", NULL)) { + log_error("lvmetad message failed."); + daemon_reply_destroy(reply); + return_0; + } + + if ((cn = dm_config_find_node(reply.cft->root, "physical_volumes"))) { + for (cn = cn->child; cn; cn = cn->sib) { + if (!(pvcl = dm_pool_zalloc(cmd->mem, sizeof(*pvcl)))) { + log_error("pv_cache_list allocation failed."); + return 0; + } + + pvid_txt = cn->key; + if (!id_read_format(&pvcl->pvid, pvid_txt)) { + stack; + continue; + } + + pvcl->devt = dm_config_find_int(cn->child, "device", 0); + + if ((vgid = dm_config_find_str(cn->child, "vgid", NULL))) + pvcl->vgid = dm_pool_strdup(cmd->mem, vgid); + + dm_list_add(pvc_list, &pvcl->list); + } + } + + daemon_reply_destroy(reply); + + return 1; +} + +/* + * Opening the device RDWR should trigger a udev db update. + * FIXME: is there a better way to update the udev db than + * doing an open/close of the device? + */ +static void _update_pv_in_udev(struct cmd_context *cmd, dev_t devt) +{ + struct device *dev; + + log_debug_devs("device %d:%d open to update udev", + (int)MAJOR(devt), (int)MINOR(devt)); + + if (!(dev = dev_cache_get_by_devt(devt, cmd->lvmetad_filter))) { + log_error("_update_pv_in_udev no dev found"); + return; + } + + if (!dev_open(dev)) + return; + dev_close(dev); +} + +/* + * Compare before and after PV lists from before/after rescanning, + * and update udev db for changes. + * + * For PVs that have changed pvid or vgid in lvmetad from rescanning, + * there may be information in the udev database to update, so open + * these devices to trigger a udev update. + * + * "before" refers to the list of pvs from lvmetad before rescanning + * "after" refers to the list of pvs from lvmetad after rescanning + * + * Comparing both lists, we can see which PVs changed (pvid or vgid), + * and trigger a udev db update for those. + */ +static void _update_changed_pvs_in_udev(struct cmd_context *cmd, + struct dm_list *pvc_before, + struct dm_list *pvc_after) +{ + struct pv_cache_list *before; + struct pv_cache_list *after; + char id_before[ID_LEN + 1] __attribute__((aligned(8))); + char id_after[ID_LEN + 1] __attribute__((aligned(8))); + int found; + + dm_list_iterate_items(before, pvc_before) { + found = 0; + + dm_list_iterate_items(after, pvc_after) { + if (after->found) + continue; + + if (before->devt != after->devt) + continue; + + if (!id_equal(&before->pvid, &after->pvid)) { + memset(id_before, 0, sizeof(id_before)); + memset(id_after, 0, sizeof(id_after)); + strncpy(&id_before[0], (char *) &before->pvid, sizeof(id_before) - 1); + strncpy(&id_after[0], (char *) &after->pvid, sizeof(id_after) - 1); + + log_debug_devs("device %d:%d changed pvid from %s to %s", + (int)MAJOR(before->devt), (int)MINOR(before->devt), + id_before, id_after); + + before->update_udev = 1; + + } else if ((before->vgid && !after->vgid) || + (after->vgid && !before->vgid) || + (before->vgid && after->vgid && strcmp(before->vgid, after->vgid))) { + + log_debug_devs("device %d:%d changed vg from %s to %s", + (int)MAJOR(before->devt), (int)MINOR(before->devt), + before->vgid ?: "none", after->vgid ?: "none"); + + before->update_udev = 1; + } + + after->found = 1; + before->found = 1; + found = 1; + break; + } + + if (!found) { + memset(id_before, 0, sizeof(id_before)); + strncpy(&id_before[0], (char *) &before->pvid, sizeof(id_before) - 1); + + log_debug_devs("device %d:%d pvid %s vg %s is gone", + (int)MAJOR(before->devt), (int)MINOR(before->devt), + id_before, before->vgid ? before->vgid : "none"); + + before->update_udev = 1; + } + } + + dm_list_iterate_items(before, pvc_before) { + if (before->update_udev) + _update_pv_in_udev(cmd, before->devt); + } + + dm_list_iterate_items(after, pvc_after) { + if (after->update_udev) + _update_pv_in_udev(cmd, after->devt); + } +} + +/* + * Before this command was run, some external entity may have + * invalidated lvmetad's cache of global information, e.g. lvmlockd. + * + * The global information includes things like a new VG, a + * VG that was removed, the assignment of a PV to a VG; + * any change that is not isolated within a single VG. + * + * The external entity, like a lock manager, would invalidate + * the lvmetad global cache if it detected that the global + * information had been changed on disk by something other + * than a local lvm command, e.g. an lvm command on another + * host with access to the same devices. (How it detects + * the change is specific to lock manager or other entity.) + * + * The effect is that metadata on disk is newer than the metadata + * in the local lvmetad daemon, and the local lvmetad's cache + * should be updated from disk before this command uses it. + * + * So, using this function, a command checks if lvmetad's global + * cache is valid. If so, it does nothing. If not, it rescans + * devices to update the lvmetad cache, then it notifies lvmetad + * that it's cache is valid again (consistent with what's on disk.) + * This command can then go ahead and use the newly refreshed metadata. + * + * 1. Check if the lvmetad global cache is invalid. + * 2. If so, reread metadata from all devices and update the lvmetad cache. + * 3. Tell lvmetad that the global cache is now valid. + */ + +void lvmetad_validate_global_cache(struct cmd_context *cmd, int force) +{ + struct dm_list pvc_before; /* pv_cache_list */ + struct dm_list pvc_after; /* pv_cache_list */ + daemon_reply reply; + int global_invalid; + + dm_list_init(&pvc_before); + dm_list_init(&pvc_after); + + if (!lvmetad_used()) + return; + + if (force) + goto do_scan; + + reply = daemon_send_simple(_lvmetad, "get_global_info", + "token = %s", "skip", + NULL); + + if (reply.error) { + log_error("lvmetad_validate_global_cache get_global_info error %d", reply.error); + goto do_scan; + } + + if (strcmp(daemon_reply_str(reply, "response", ""), "OK")) { + log_error("lvmetad_validate_global_cache get_global_info not ok"); + goto do_scan; + } + + global_invalid = daemon_reply_int(reply, "global_invalid", -1); + + daemon_reply_destroy(reply); + + if (!global_invalid) { + /* cache is valid */ + return; + } + + do_scan: + /* + * Save the current state of pvs from lvmetad so after devices are + * scanned, we can compare to the new state to see if pvs changed. + */ + _lvmetad_get_pv_cache_list(cmd, &pvc_before); + + /* + * Update the local lvmetad cache so it correctly reflects any + * changes made on remote hosts. + */ + lvmetad_pvscan_all_devs(cmd, NULL); + + /* + * Clear the global_invalid flag in lvmetad. + * Subsequent local commands that read global state + * from lvmetad will not see global_invalid until + * another host makes another global change. + */ + reply = daemon_send_simple(_lvmetad, "set_global_info", + "token = %s", "skip", + "global_invalid = %d", 0, + NULL); + if (reply.error) + log_error("lvmetad_validate_global_cache set_global_info error %d", reply.error); + + if (strcmp(daemon_reply_str(reply, "response", ""), "OK")) + log_error("lvmetad_validate_global_cache set_global_info not ok"); + + daemon_reply_destroy(reply); + + /* + * Populate this command's lvmcache structures from lvmetad. + */ + lvmcache_seed_infos_from_lvmetad(cmd); + + /* + * Update the local udev database to reflect PV changes from + * other hosts. + * + * Compare the before and after PV lists, and if a PV's + * pvid or vgid has changed, then open that device to trigger + * a uevent to update the udev db. + * + * This has no direct benefit to lvm, but is just a best effort + * attempt to keep the udev db updated and reflecting current + * lvm information. + * + * FIXME: lvmcache_seed_infos_from_lvmetad() and _lvmetad_get_pv_cache_list() + * each get pv_list from lvmetad, and they could share a single pv_list reply. + */ + if (!dm_list_empty(&pvc_before)) { + _lvmetad_get_pv_cache_list(cmd, &pvc_after); + _update_changed_pvs_in_udev(cmd, &pvc_before, &pvc_after); + } +} diff --git a/lib/cache/lvmetad.h b/lib/cache/lvmetad.h index 822467500..af0d562fe 100644 --- a/lib/cache/lvmetad.h +++ b/lib/cache/lvmetad.h @@ -167,6 +167,7 @@ int lvmetad_pvscan_all_devs(struct cmd_context *cmd, activation_handler handler) int lvmetad_pvscan_foreign_vgs(struct cmd_context *cmd, activation_handler handler); int lvmetad_vg_clear_outdated_pvs(struct volume_group *vg); +void lvmetad_validate_global_cache(struct cmd_context *cmd, int force); # else /* LVMETAD_SUPPORT */ @@ -195,6 +196,7 @@ int lvmetad_vg_clear_outdated_pvs(struct volume_group *vg); # define lvmetad_pvscan_all_devs(cmd, handler) (0) # define lvmetad_pvscan_foreign_vgs(cmd, handler) (0) # define lvmetad_vg_clear_outdated_pvs(vg) (1) +# define lvmetad_validate_global_cache(cmd, force) do { } while (0) # endif /* LVMETAD_SUPPORT */