lvmdbusd: Set LVM_COMMAND_PROFILE=lvmdbusd

We need this to prevent lvm from interleaving the JSON output with errors
written to stderr.

(cherry picked from commit a5e6947d74)

Makefile.in

@@ -110,7 +110,7 @@ rpm: dist
 	$(LN_S) -f $(abs_top_srcdir)/spec/build.inc $(rpmbuilddir)/SOURCES
 	$(LN_S) -f $(abs_top_srcdir)/spec/macros.inc $(rpmbuilddir)/SOURCES
 	$(LN_S) -f $(abs_top_srcdir)/spec/packages.inc $(rpmbuilddir)/SOURCES
-	DM_VER=$$(cut -d- -f1 $(top_srcdir)/VERSION_DM);\
+	DM_VER=$$(cut -d' ' -f1 $(top_srcdir)/VERSION_DM | cut -d- -f1);\
 	GIT_VER=$$(cd $(top_srcdir); git describe | cut -d- --output-delimiter=. -f2,3 || echo 0);\
 	$(SED) -e "s,\(device_mapper_version\) [0-9.]*$$,\1 $$DM_VER," \
 	    -e "s,^\(Version:[^0-9%]*\)[0-9.]*$$,\1 $(LVM_VER)," \

WHATS_NEW

@@ -1,3 +1,7 @@
+Version 2.03.17 - 
+===============================
+  Fix lvconvert --test --type vdo-pool execution.
+
 Version 2.03.14 - 20th October 2021
 ===================================
   Device scanning is skipping directories on different filesystems.

daemons/lvmdbusd/main.py

@@ -112,8 +112,10 @@ def main():
 
 	use_session = os.getenv('LVMDBUSD_USE_SESSION', False)
 
-	# Ensure that we get consistent output for parsing stdout/stderr
+	# Ensure that we get consistent output for parsing stdout/stderr and that we
+	# are using the lvmdbusd profile.
 	os.environ["LC_ALL"] = "C"
+	os.environ["LVM_COMMAND_PROFILE"] = "lvmdbusd"
 
 	cfg.args = parser.parse_args()
 	cfg.create_request_entry = RequestEntry

daemons/lvmlockd/lvmlockd-sanlock.c

@@ -684,10 +684,10 @@ int lm_init_vg_sanlock(char *ls_name, char *vg_name, uint32_t flags, char *vg_ar
 			break;
 		}
 
-		if (rv) {
+		if (rv < 0) {
 			log_error("clear lv resource area %llu error %d",
 				  (unsigned long long)offset, rv);
-			break;
+			return rv;
 		}
 		offset += align_size;
 	}

device_mapper/vdo/target.h

@@ -77,8 +77,10 @@ enum dm_vdo_write_policy {
 struct dm_vdo_target_params {
 	uint32_t minimum_io_size;       // in sectors
 	uint32_t block_map_cache_size_mb;
-	uint32_t block_map_era_length;	// format period
-
+	union {
+		uint32_t block_map_era_length;	// format period
+		uint32_t block_map_period;      // supported alias
+	};
 	uint32_t check_point_frequency;
 	uint32_t index_memory_size_mb;  // format
 

lib/Makefile.in

@@ -40,6 +40,7 @@ SOURCES =\
 	device/dev-luks.c \
 	device/dev-dasd.c \
 	device/dev-lvm1-pool.c \
+	device/parse_vpd.c \
 	display/display.c \
 	error/errseg.c \
 	unknown/unknown.c \

lib/activate/dev_manager.c

@@ -2875,6 +2875,10 @@ int add_areas_line(struct dev_manager *dm, struct lv_segment *seg,
 
 	/* FIXME Avoid repeating identical stat in dm_tree_node_add_target_area */
 	for (s = start_area; s < areas; s++) {
+
+		/* FIXME: dev_name() does not return NULL!  It needs to check if dm_list_empty(&dev->aliases)
+		   but this knot of logic is too complex to pull apart without careful deconstruction. */
+
 		if ((seg_type(seg, s) == AREA_PV &&
 		     (!seg_pvseg(seg, s) || !seg_pv(seg, s) || !seg_dev(seg, s) ||
 		       !(name = dev_name(seg_dev(seg, s))) || !*name ||
@@ -2893,7 +2897,10 @@ int add_areas_line(struct dev_manager *dm, struct lv_segment *seg,
 				return_0;
 			num_error_areas++;
 		} else if (seg_type(seg, s) == AREA_PV) {
-			if (!dm_tree_node_add_target_area(node, dev_name(seg_dev(seg, s)), NULL,
+			struct device *dev = seg_dev(seg, s);
+			name = dm_list_empty(&dev->aliases) ? NULL : dev_name(dev);
+
+			if (!dm_tree_node_add_target_area(node, name, NULL,
 				    (seg_pv(seg, s)->pe_start + (extent_size * seg_pe(seg, s)))))
 				return_0;
 			num_existing_areas++;

lib/cache/lvmcache.c

@@ -625,6 +625,151 @@ static void _warn_unused_duplicates(struct cmd_context *cmd)
 	}
 }
 
+static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_info *info, const char *pvid,
+				     struct dm_list *altdevs, struct device **dev_mpath)
+{
+	struct device_list *devl;
+	struct device *dev_mp = NULL;
+	struct device *dev1 = NULL;
+	struct device *dev;
+	const char *wwid1 = NULL;
+	const char *wwid;
+	int diff_wwid = 0;
+	int same_wwid = 0;
+	int dev_is_mp;
+
+	*dev_mpath = NULL;
+
+	if (!find_config_tree_bool(cmd, devices_multipath_component_detection_CFG, NULL))
+		return 0;
+
+	/* This function only makes sense with more than one dev. */
+	if ((info && dm_list_empty(altdevs)) || (!info && (dm_list_size(altdevs) == 1))) {
+		log_debug("Skip multipath component checks with single device for PVID %s", pvid);
+		return 0;
+	}
+
+	log_debug("Checking for multipath components for duplicate PVID %s", pvid);
+
+	if (info) {
+		dev = info->dev;
+		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
+
+		if (dev_is_mp) {
+			if ((wwid1 = dev_mpath_component_wwid(cmd, dev))) {
+				dev_mp = dev;
+				dev1 = dev;
+			}
+		} else {
+			if ((wwid1 = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID)))
+				dev1 = dev;
+		}
+	}
+
+	dm_list_iterate_items(devl, altdevs) {
+		dev = devl->dev;
+		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
+
+		if (dev_is_mp)
+			wwid = dev_mpath_component_wwid(cmd, dev);
+		else
+			wwid = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID);
+
+		if (!wwid && wwid1) {
+			log_debug("Different wwids for duplicate PVs %s %s %s none",
+				  dev_name(dev1), wwid1, dev_name(dev));
+			diff_wwid++;
+			continue;
+		}
+
+		if (!wwid)
+			continue;
+
+		if (!wwid1) {
+			wwid1 = wwid;
+			dev1 = dev;
+			continue;
+		}
+
+		/* Different wwids indicates these are not multipath components. */
+		if (strcmp(wwid1, wwid)) {
+			log_debug("Different wwids for duplicate PVs %s %s %s %s",
+				  dev_name(dev1), wwid1, dev_name(dev), wwid);
+			diff_wwid++;
+			continue;
+		}
+
+		/* Different mpath devs with the same wwid shouldn't happen. */
+		if (dev_is_mp && dev_mp) {
+			log_print("Found multiple multipath devices for PVID %s WWID %s: %s %s",
+				   pvid, wwid1, dev_name(dev_mp), dev_name(dev));
+			continue;
+		}
+
+		log_debug("Same wwids for duplicate PVs %s %s", dev_name(dev1), dev_name(dev));
+		same_wwid++;
+
+		/* Save the mpath device so it can be used as the PV. */
+		if (dev_is_mp)
+			dev_mp = dev;
+	}
+
+	if (diff_wwid || !same_wwid)
+		return 0;
+
+	if (dev_mp)
+		log_debug("Found multipath device %s for PVID %s WWID %s.", dev_name(dev_mp), pvid, wwid1);
+
+	*dev_mpath = dev_mp;
+	return 1;
+}
+
+static int _all_md_components(struct cmd_context *cmd, struct lvmcache_info *info, const char *pvid,
+			      struct dm_list *altdevs, struct device **dev_md_out)
+{
+	struct device_list *devl;
+	struct device *dev_md = NULL;
+	struct device *dev;
+	int real_dup = 0;
+ 
+	*dev_md_out = NULL;
+
+	/* There will often be no info struct because of the extra_md_checks function. */
+ 
+	if (info && (cmd->dev_types->md_major == MAJOR(info->dev->dev)))
+		dev_md = info->dev;
+ 
+	dm_list_iterate_items(devl, altdevs) {
+		dev = devl->dev;
+ 
+		if (cmd->dev_types->md_major == MAJOR(dev->dev)) {
+			if (dev_md) {
+				/* md devs themselves are dups */
+				log_debug("Found multiple md devices for PVID %s: %s %s",
+					  pvid, dev_name(dev_md), dev_name(dev));
+				real_dup = 1;
+				break;
+			} else
+				dev_md = dev;
+		} else {
+			if (!dev_is_md_component(cmd, dev, NULL, 1)) {
+				/* md dev copied to another device */
+				real_dup = 1;
+				break;
+			}
+		}
+	}
+ 
+	if (real_dup)
+		return 0;
+ 
+	if (dev_md)
+		log_debug("Found md device %s for PVID %s.", dev_name(dev_md), pvid);
+ 
+	*dev_md_out = dev_md;
+	return 1;
+}
+
 /*
  * If we've found devices with the same PVID, decide which one
  * to use.
@@ -680,6 +825,8 @@ static void _choose_duplicates(struct cmd_context *cmd,
 	struct device_list *devl, *devl_safe, *devl_add, *devl_del;
 	struct lvmcache_info *info;
 	struct device *dev1, *dev2;
+	struct device *dev_mpath, *dev_md;
+	struct device *dev_drop;
 	const char *device_id = NULL, *device_id_type = NULL;
 	const char *idname1 = NULL, *idname2 = NULL;
 	uint32_t dev1_major, dev1_minor, dev2_major, dev2_minor;
@@ -702,6 +849,8 @@ static void _choose_duplicates(struct cmd_context *cmd,
 next:
 	dm_list_init(&altdevs);
 	pvid = NULL;
+	dev_mpath = NULL;
+	dev_md = NULL;
 
 	dm_list_iterate_items_safe(devl, devl_safe, &_initial_duplicates) {
 		if (!pvid) {
@@ -720,30 +869,173 @@ next:
 		return;
 	}
 
+	info = lvmcache_info_from_pvid(pvid, NULL, 0);
+
 	/*
-	 * Get rid of any md components before comparing alternatives.
-	 * (Since an md component can never be used, it's not an
-	 * option to use like other kinds of alternatives.)
+	 * Usually and ideally, components of md and multipath devs should have
+	 * been excluded by filters, and not scanned for a PV.  In some unusual
+	 * cases the components can get through the filters, and a PV can be
+	 * found on them.  Detecting the same PVID on both the component and
+	 * the md/mpath device gives us a last chance to drop the component.
+	 * An md/mpath component device is completely ignored, as if it had
+	 * been filtered, and not kept in the list unused duplicates.
+	 *
+	 * One issue related to eliminating mpath/md duplicate PVs here is
+	 * that it occurs after label_scan, and hints are created based
+	 * on what label_scan finds, so hints are disabled due to duplicate
+	 * PVs that are later resolved here.
 	 */
 
-	info = lvmcache_info_from_pvid(pvid, NULL, 0);
-	if (info && dev_is_md_component(cmd, info->dev, NULL, 1)) {
-		/* does not go in del_cache_devs which become unused_duplicates */
-		log_debug_cache("PV %s drop MD component from scan selection %s", pvid, dev_name(info->dev));
-		lvmcache_del(info);
-		info = NULL;
-	}
+	/*
+	 * Get rid of multipath components based on matching wwids.
+	 */
+	if (_all_multipath_components(cmd, info, pvid, &altdevs, &dev_mpath)) {
+		if (info && dev_mpath && (info->dev != dev_mpath)) {
+			/*
+			 * info should be dropped from lvmcache and info->dev
+			 * should be treated as if it had been excluded by a filter.
+			 * dev_mpath should be added to lvmcache by the caller.
+			 */
+			dev_drop = info->dev;
 
-	dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
-		if (dev_is_md_component(cmd, devl->dev, NULL, 1)) {
-			log_debug_cache("PV %s drop MD component from scan duplicates %s", pvid, dev_name(devl->dev));
-			dm_list_del(&devl->list);
+			/* Have caller add dev_mpath to lvmcache. */
+			log_debug("Using multipath device %s for PVID %s.", dev_name(dev_mpath), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_mpath;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_mpath from altdevs. */
+			if ((devl = _get_devl_in_device_list(dev_mpath, &altdevs)))
+				dm_list_del(&devl->list);
+
+			/* Remove info from lvmcache that came from the component dev. */
+			log_debug("Ignoring multipath component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
 		}
+
+		if (info && !dev_mpath) {
+			/*
+			 * Only mpath component devs were found and no actual
+			 * multipath dev, so drop the component from lvmcache.
+			 */
+			dev_drop = info->dev;
+
+			log_debug("Ignoring multipath component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
+			/*
+			 * The altdevs are all mpath components that should look
+			 * like they were filtered, they are not in lvmcache.
+			 */
+			dev_drop = devl->dev;
+
+			log_debug("Ignoring multipath component %s with PVID %s (dropping duplicate)", dev_name(dev_drop), pvid);
+			dm_list_del(&devl->list);
+
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+		goto next;
 	}
 
-	if (dm_list_empty(&altdevs))
-		goto next;
+	/*
+	 * Get rid of any md components.
+	 */
+	if (_all_md_components(cmd, info, pvid, &altdevs, &dev_md)) {
+		if (info && dev_md && (info->dev != dev_md)) {
+			/*
+			 * info should be dropped from lvmcache and info->dev
+			 * should be treated as if it had been excluded by a filter.
+			 * dev_md should be added to lvmcache by the caller.
+			 * Often this info struct has been removed by
+			 * lvmcache_extra_md_component_checks.
+			 */
+			dev_drop = info->dev;
 
+			/* Have caller add dev_md to lvmcache. */
+			log_debug("Using md device %s for PVID %s.", dev_name(dev_md), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_md;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_md from altdevs. */
+			if ((devl = _get_devl_in_device_list(dev_md, &altdevs)))
+				dm_list_del(&devl->list);
+
+			/* Remove info from lvmcache that came from the component dev. */
+			log_debug("Ignoring md component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		if (!info && dev_md) {
+			/*
+			 * The info struct was from a component and was dropped
+			 * and the actual md dev was found on initial_duplicates
+			 * and the caller should add it to lvmcache.
+			 */
+
+			/* Have caller add dev_md to lvmcache. */
+			log_debug("Using md device %s for PVID %s.", dev_name(dev_md), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_md;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_md from altdevs. */
+			if ((devl = _get_devl_in_device_list(dev_md, &altdevs)))
+				dm_list_del(&devl->list);
+		}
+
+		if (info && !dev_md) {
+			/*
+			 * Only md component devs were found and no actual
+			 * md dev, so drop the component from lvmcache.
+			 */
+			dev_drop = info->dev;
+
+			log_debug("Ignoring md component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
+			/*
+			 * The altdevs are all md components that should look
+			 * like they were filtered, they are not in lvmcache.
+			 */
+			dev_drop = devl->dev;
+
+			log_debug("Ignoring md component %s with PVID %s (dropping duplicate)", dev_name(dev_drop), pvid);
+			dm_list_del(&devl->list);
+
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+		goto next;
+	}
 
 	/*
 	 * Find the device for the pvid that's currently in lvmcache.
@@ -1149,6 +1441,18 @@ int lvmcache_label_reopen_vg_rw(struct cmd_context *cmd, const char *vgname, con
  * times it can be a clue that label_scan mistakenly read the pv from an md
  * component device instead of from the md device itself.  So for unmatching
  * sizes, we do a full md component check on the device.
+ *
+ * It might be nice to do this checking in the filter (when passes_filter is
+ * called after the initial read), but that doesn't work because passes_filter
+ * is called before _text_read so metadata/pvsummary info is not yet available
+ * which this function uses.
+ *
+ * The unique value of this function is that it can eliminate md components
+ * without there being duplicate PVs.  But, there will often be duplicate PVs,
+ * handled by _all_md_components(), where other devs with the same pvid will be
+ * in _initial_duplicates.  One could be the md device itself which will be
+ * added to lvmcache by choose_duplicates, and other duplicates that are
+ * components will be dropped.
  */
 
 void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
@@ -1210,7 +1514,8 @@ void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
 			 */
 			if (pvsize && devsize && (pvsize != devsize))
 				do_check_size = 1;
-			if (device_hint && !strncmp(device_hint, "/dev/md", 7))
+			if (device_hint && !strncmp(device_hint, "/dev/md", 7) &&
+			    (MAJOR(info->dev->dev) != cmd->dev_types->md_major))
 				do_check_name = 1;
 
 			if (!do_check_size && !do_check_name)
@@ -1240,11 +1545,11 @@ void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
 				  device_hint ?: "none", dev_name(dev));
 
 			if (dev_is_md_component(cmd, dev, NULL, 1)) {
-				log_debug("dropping PV from md component %s", dev_name(dev));
+				log_debug("Ignoring PV from md component %s with PVID %s (metadata %s %llu)",
+					  dev_name(dev), dev->pvid, device_hint ?: "none", (unsigned long long)pvsize);
 				dev->flags &= ~DEV_SCAN_FOUND_LABEL;
 				/* lvmcache_del will also delete vginfo if info was last one */
 				lvmcache_del(info);
-				lvmcache_del_dev_from_duplicates(dev);
 				cmd->filter->wipe(cmd, cmd->filter, dev, NULL);
 			}
 		}
@@ -1298,7 +1603,8 @@ int lvmcache_label_scan(struct cmd_context *cmd)
 	 * with infos/vginfos based on reading headers from
 	 * each device, and a vg summary from each mda.
 	 */
-	label_scan(cmd);
+	if (!label_scan(cmd))
+		return_0;
 
 	/*
 	 * When devnames are used as device ids (which is dispreferred),

lib/commands/toolcontext.c

@@ -1143,19 +1143,6 @@ static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 	 * Update MAX_FILTERS definition above when adding new filters.
 	 */
 
-	/*
-	 * sysfs filter. Only available on 2.6 kernels.  Non-critical.
-	 * Listed first because it's very efficient at eliminating
-	 * unavailable devices.
-	 *
-	 * TODO: I suspect that using the lvm_type and device_id
-	 * filters before this one may be more efficient.
-	 */
-	if (find_config_tree_bool(cmd, devices_sysfs_scan_CFG, NULL)) {
-		if ((filters[nr_filt] = sysfs_filter_create()))
-			nr_filt++;
-	}
-
 	/* internal filter used by command processing. */
 	if (!(filters[nr_filt] = internal_filter_create())) {
 		log_error("Failed to create internal device filter");
@@ -1195,6 +1182,17 @@ static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 	}
 	nr_filt++;
 
+	/*
+	 * sysfs filter. Only available on 2.6 kernels.  Non-critical.
+	 * Eliminates unavailable devices.
+	 * TODO: this may be unnecessary now with device ids
+	 * (currently not used for devs match to device id using syfs)
+	 */
+	if (find_config_tree_bool(cmd, devices_sysfs_scan_CFG, NULL)) {
+		if ((filters[nr_filt] = sysfs_filter_create()))
+			nr_filt++;
+	}
+
 	/* usable device filter. Required. */
 	if (!(filters[nr_filt] = usable_filter_create(cmd, cmd->dev_types, FILTER_MODE_NO_LVMETAD))) {
 		log_error("Failed to create usabled device filter");
@@ -1605,7 +1603,6 @@ struct cmd_context *create_config_context(void)
 
 	dm_list_init(&cmd->config_files);
 	dm_list_init(&cmd->tags);
-	dm_list_init(&cmd->hints);
 
 	if (!_init_lvm_conf(cmd))
 		goto_out;
@@ -1670,7 +1667,6 @@ struct cmd_context *create_toolcontext(unsigned is_clvmd,
 	dm_list_init(&cmd->formats);
 	dm_list_init(&cmd->segtypes);
 	dm_list_init(&cmd->tags);
-	dm_list_init(&cmd->hints);
 	dm_list_init(&cmd->config_files);
 	label_init();
 
@@ -1916,7 +1912,6 @@ int refresh_toolcontext(struct cmd_context *cmd)
 	_destroy_segtypes(&cmd->segtypes);
 	_destroy_formats(cmd, &cmd->formats);
 
-	devices_file_exit(cmd);
 	if (!dev_cache_exit())
 		stack;
 	_destroy_dev_types(cmd);

lib/commands/toolcontext.h

@@ -206,7 +206,6 @@ struct cmd_context {
 	 * Devices and filtering.
 	 */
 	struct dev_filter *filter;
-	struct dm_list hints;
 	struct dm_list use_devices;		/* struct dev_use for each entry in devices file */
 	const char *md_component_checks;
 	const char *search_for_devnames;	/* config file setting */

lib/device/dev-cache.c

@@ -80,6 +80,7 @@ static void _dev_init(struct device *dev)
 
 	dm_list_init(&dev->aliases);
 	dm_list_init(&dev->ids);
+	dm_list_init(&dev->wwids);
 }
 
 void dev_destroy_file(struct device *dev)
@@ -351,7 +352,7 @@ static int _add_alias(struct device *dev, const char *path, enum add_hash hash)
 			goto out;
 		}
 
-	if (!(path = dm_pool_strdup(_cache.mem, path)) ||
+	if (!(path = _strdup(path)) ||
 	    !(sl = _zalloc(sizeof(*sl)))) {
 		log_error("Failed to add allias to dev cache.");
 		return 0;
@@ -383,6 +384,22 @@ out:
 	return 1;
 }
 
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen)
+{
+	int ret;
+	int fd;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return 0;
+	ret = read(fd, buf, buf_size);
+	close(fd);
+	if (ret <= 0)
+		return 0;
+	*retlen = ret;
+	return 1;
+}
+
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value)
 {
 	FILE *fp;
@@ -1162,6 +1179,17 @@ static int _insert(const char *path, const struct stat *info,
 	return 1;
 }
 
+static void _drop_all_aliases(struct device *dev)
+{
+	struct dm_str_list *strl, *strl2;
+
+	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
+		log_debug("Drop alias for %d:%d %s.", (int)MAJOR(dev->dev), (int)MINOR(dev->dev), strl->str);
+		dm_hash_remove(_cache.names, strl->str);
+		dm_list_del(&strl->list);
+	}
+}
+
 void dev_cache_scan(struct cmd_context *cmd)
 {
 	log_debug_devs("Creating list of system devices.");
@@ -1325,6 +1353,7 @@ int dev_cache_exit(void)
 		dm_hash_iterate(n, _cache.names) {
 			dev = (struct device *) dm_hash_get_data(_cache.names, n);
 			free_dids(&dev->ids);
+			free_wwids(&dev->wwids);
 		}
 	}
 
@@ -1371,59 +1400,6 @@ int dev_cache_add_dir(const char *path)
 	return 1;
 }
 
-/* Check cached device name is still valid before returning it */
-/* This should be a rare occurrence */
-/* set quiet if the cache is expected to be out-of-date */
-/* FIXME Make rest of code pass/cache struct device instead of dev_name */
-const char *dev_name_confirmed(struct device *dev, int quiet)
-{
-	struct stat buf;
-	const char *name;
-	int r;
-
-	if ((dev->flags & DEV_REGULAR))
-		return dev_name(dev);
-
-	while ((r = stat(name = dm_list_item(dev->aliases.n,
-					  struct dm_str_list)->str, &buf)) ||
-	       (buf.st_rdev != dev->dev)) {
-		if (r < 0) {
-			if (quiet)
-				log_sys_debug("stat", name);
-			else
-				log_sys_error("stat", name);
-		}
-		if (quiet)
-			log_debug_devs("Path %s no longer valid for device(%d,%d)",
-				       name, (int) MAJOR(dev->dev),
-				       (int) MINOR(dev->dev));
-		else
-			log_warn("Path %s no longer valid for device(%d,%d)",
-				 name, (int) MAJOR(dev->dev),
-				 (int) MINOR(dev->dev));
-
-		/* Remove the incorrect hash entry */
-		dm_hash_remove(_cache.names, name);
-
-		/* Leave list alone if there isn't an alternative name */
-		/* so dev_name will always find something to return. */
-		/* Otherwise add the name to the correct device. */
-		if (dm_list_size(&dev->aliases) > 1) {
-			dm_list_del(dev->aliases.n);
-			if (!r)
-				_insert(name, &buf, 0, obtain_device_list_from_udev());
-			continue;
-		}
-
-		/* Scanning issues this inappropriately sometimes. */
-		log_debug_devs("Aborting - please provide new pathname for what "
-			       "used to be %s", name);
-		return NULL;
-	}
-
-	return dev_name(dev);
-}
-
 struct device *dev_hash_get(const char *name)
 {
 	return (struct device *) dm_hash_lookup(_cache.names, name);
@@ -1452,26 +1428,23 @@ static void _remove_alias(struct device *dev, const char *name)
  * deactivated LV.  Those old paths are all invalid and are dropped here.
  */
 
-static void _verify_aliases(struct device *dev, const char *newname)
+void dev_cache_verify_aliases(struct device *dev)
 {
 	struct dm_str_list *strl, *strl2;
 	struct stat st;
 
 	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
-		/* newname was just stat'd and added by caller */
-		if (newname && !strcmp(strl->str, newname))
-			continue;
-
 		if (stat(strl->str, &st) || (st.st_rdev != dev->dev)) {
-			log_debug("Drop invalid path %s for %d:%d (new path %s).",
-				  strl->str, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), newname ?: "");
+			log_debug("Drop alias for %d:%d invalid path %s %d:%d.",
+				  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), strl->str,
+				  (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev));
 			dm_hash_remove(_cache.names, strl->str);
 			dm_list_del(&strl->list);
 		}
 	}
 }
 
-struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f)
+static struct device *_dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f, int existing)
 {
 	struct device *dev = (struct device *) dm_hash_lookup(_cache.names, name);
 	struct stat st;
@@ -1485,13 +1458,18 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	if (dev && (dev->flags & DEV_REGULAR))
 		return dev;
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
 	/*
-	 * The requested path is invalid, remove any dev-cache
-	 * info for it.
+	 * The requested path is invalid, remove any dev-cache info for it.
 	 */
 	if (stat(name, &st)) {
 		if (dev) {
-			log_print("Device path %s is invalid for %d:%d %s.",
+			log_debug("Device path %s is invalid for %d:%d %s.",
 				  name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev));
 
 			dm_hash_remove(_cache.names, name);
@@ -1499,11 +1477,17 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 			_remove_alias(dev, name);
 
 			/* Remove any other names in dev->aliases that are incorrect. */
-			_verify_aliases(dev, NULL);
+			dev_cache_verify_aliases(dev);
 		}
 		return NULL;
 	}
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
 	if (!S_ISBLK(st.st_mode)) {
 		log_debug("Not a block device %s.", name);
 		return NULL;
@@ -1514,26 +1498,41 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	 * Remove incorrect info and then add new dev-cache entry.
 	 */
 	if (dev && (st.st_rdev != dev->dev)) {
-		log_debug("Device path %s does not match %d:%d %s.",
-			  name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev));
+		struct device *dev_by_devt = (struct device *) btree_lookup(_cache.devices, (uint32_t) st.st_rdev);
 
-		dm_hash_remove(_cache.names, name);
+		/*
+		 * lvm commands create this condition when they
+		 * activate/deactivate LVs combined with creating new LVs.
+		 * The command does not purge dev structs when deactivating
+		 * an LV (which it probably should do), but the better
+		 * approach would be not using dev-cache at all for LVs.
+		 */
 
-		_remove_alias(dev, name);
+		log_debug("Dropping aliases for device entry %d:%d %s for new device %d:%d %s.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev),
+			  (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev), name);
 
-		/* Remove any other names in dev->aliases that are incorrect. */
-		_verify_aliases(dev, NULL);
+		_drop_all_aliases(dev);
 
-		/* Add new dev-cache entry next. */
-		dev = NULL;
-	}
+		if (dev_by_devt) {
+			log_debug("Dropping aliases for device entry %d:%d %s for new device %d:%d %s.",
+				   (int)MAJOR(dev_by_devt->dev), (int)MINOR(dev_by_devt->dev), dev_name(dev_by_devt),
+				   (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev), name);
+
+			_drop_all_aliases(dev_by_devt);
+		}
+
+#if 0
+		/*
+		 * I think only lvm's own dm devs should be added here, so use
+		 * a warning to look for any other unknown cases.
+		 */
+		if (MAJOR(st.st_rdev) != cmd->dev_types->device_mapper_major) {
+			log_warn("WARNING: new device appeared %d:%d %s",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+		}
+#endif
 
-	/*
-	 * Either add a new struct dev for st_rdev and name,
-	 * or add name as a new alias for an existing struct dev
-	 * for st_rdev.
-	 */
-	if (!dev) {
 		if (!_insert_dev(name, st.st_rdev))
 			return_NULL;
 
@@ -1545,9 +1544,77 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 			return NULL;
 		}
 
-		_verify_aliases(dev, name);
+		goto out;
 	}
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
+	if (!dev && existing)
+		return_NULL;
+
+	/*
+	 * This case should never be hit for a PV. It should only
+	 * happen when the command is opening a new LV it has created.
+	 * Add an arg to all callers indicating when the arg should be
+	 * new (for an LV) and not existing.
+	 * FIXME: fix this further by not using dev-cache struct devs
+	 * at all for new dm devs (LVs) that lvm uses.  Make the
+	 * dev-cache contain only devs for PVs.
+	 * Places to fix that use a dev for LVs include:
+	 * . lv_resize opening lv to discard
+	 * . wipe_lv opening lv to zero it
+	 * . _extend_sanlock_lv opening lv to extend it
+	 * . _write_log_header opening lv to write header
+	 * Also, io to LVs should not go through bcache.
+	 * bcache should contain only labels and metadata
+	 * scanned from PVs.
+	 */
+	if (!dev) {
+		/*
+		 * This case should only be used for new devices created by this
+		 * command (opening LVs it's created), so if a dev exists for the
+		 * dev_t referenced by the name, then drop all aliases for before
+		 * _insert_dev adds the new name.  lvm commands actually hit this
+		 * fairly often when it uses some LV, deactivates the LV, then
+		 * creates some new LV which ends up with the same major:minor.
+		 * Without dropping the aliases, it's plausible that lvm commands
+		 * could end up using the wrong dm device.
+		 */
+		struct device *dev_by_devt = (struct device *) btree_lookup(_cache.devices, (uint32_t) st.st_rdev);
+		if (dev_by_devt) {
+			log_debug("Dropping aliases for %d:%d before adding new path %s.",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+			_drop_all_aliases(dev_by_devt);
+		}
+
+#if 0
+		/*
+		 * I think only lvm's own dm devs should be added here, so use
+		 * a warning to look for any other unknown cases.
+		 */
+		if (MAJOR(st.st_rdev) != cmd->dev_types->device_mapper_major) {
+			log_warn("WARNING: new device appeared %d:%d %s",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+		}
+#endif
+
+		if (!_insert_dev(name, st.st_rdev))
+			return_NULL;
+
+		/* Get the struct dev that was just added. */
+		dev = (struct device *) dm_hash_lookup(_cache.names, name);
+
+		if (!dev) {
+			log_error("Failed to get device %s", name);
+			return NULL;
+		}
+	}
+
+ out:
 	/*
 	 * The caller passed a filter if they only want the dev if it
 	 * passes filters.
@@ -1577,63 +1644,23 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	return dev;
 }
 
-struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t dev, struct dev_filter *f, int *filtered)
+struct device *dev_cache_get_existing(struct cmd_context *cmd, const char *name, struct dev_filter *f)
 {
-	char path[PATH_MAX];
-	const char *sysfs_dir;
-	struct stat info;
-	struct device *d = (struct device *) btree_lookup(_cache.devices, (uint32_t) dev);
-	int ret;
+	return _dev_cache_get(cmd, name, f, 1);
+}
 
-	if (filtered)
-		*filtered = 0;
+struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f)
+{
+	return _dev_cache_get(cmd, name, f, 0);
+}
 
-	if (!d) {
-		sysfs_dir = dm_sysfs_dir();
-		if (sysfs_dir && *sysfs_dir) {
-			/* First check if dev is sysfs to avoid useless scan */
-			if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d",
-					sysfs_dir, (int)MAJOR(dev), (int)MINOR(dev)) < 0) {
-				log_error("dm_snprintf partition failed.");
-				return NULL;
-			}
-
-			if (lstat(path, &info)) {
-				log_debug("No sysfs entry for %d:%d errno %d at %s.",
-					  (int)MAJOR(dev), (int)MINOR(dev), errno, path);
-				return NULL;
-			}
-		}
-
-		log_debug_devs("Device num not found in dev_cache repeat dev_cache_scan for %d:%d",
-				(int)MAJOR(dev), (int)MINOR(dev));
-		dev_cache_scan(cmd);
-		d = (struct device *) btree_lookup(_cache.devices, (uint32_t) dev);
-
-		if (!d)
-			return NULL;
-	}
-
-	if (d->flags & DEV_REGULAR)
-		return d;
-
-	if (!f)
-		return d;
-
-	ret = f->passes_filter(cmd, f, d, NULL);
-
-	if (ret == -EAGAIN) {
-		log_debug_devs("get device by number defer filter %s", dev_name(d));
-		d->flags |= DEV_FILTER_AFTER_SCAN;
-		ret = 1;
-	}
-
-	if (ret)
-		return d;
-
-	if (filtered)
-		*filtered = 1;
+struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t devt)
+{
+	struct device *dev = (struct device *) btree_lookup(_cache.devices, (uint32_t) devt);
 
+	if (dev)
+		return dev;
+	log_debug_devs("No devno %d:%d in dev cache.", (int)MAJOR(devt), (int)MINOR(devt));
 	return NULL;
 }
 
@@ -1703,8 +1730,10 @@ int dev_fd(struct device *dev)
 
 const char *dev_name(const struct device *dev)
 {
-	return (dev && dev->aliases.n) ? dm_list_item(dev->aliases.n, struct dm_str_list)->str :
-	    unknown_device_name();
+	if (dev && dev->aliases.n && !dm_list_empty(&dev->aliases))
+		return dm_list_item(dev->aliases.n, struct dm_str_list)->str;
+	else
+		return unknown_device_name();
 }
 
 bool dev_cache_has_md_with_end_superblock(struct dev_types *dt)
@@ -1852,7 +1881,7 @@ int setup_devices_file(struct cmd_context *cmd)
  * Add all system devices to dev-cache, and attempt to
  * match all devices_file entries to dev-cache entries.
  */
-int setup_devices(struct cmd_context *cmd)
+static int _setup_devices(struct cmd_context *cmd, int no_file_match)
 {
 	int file_exists;
 	int lock_mode = 0;
@@ -1871,6 +1900,15 @@ int setup_devices(struct cmd_context *cmd)
 
 	file_exists = devices_file_exists(cmd);
 
+	/*
+	 * Fail if user specifies a file name that doesn't exist and
+	 * the command is not creating a new devices file.
+	 */
+	if (!file_exists && !cmd->create_edit_devices_file && cmd->devicesfile && strlen(cmd->devicesfile)) {
+		log_error("Devices file not found: %s", cmd->devices_file_path);
+		return 0;
+	}
+
 	/*
 	 * Removing the devices file is another way of disabling the use of
 	 * a devices file, unless the command creates the devices file.
@@ -1979,6 +2017,13 @@ int setup_devices(struct cmd_context *cmd)
 	 */
 	dev_cache_scan(cmd);
 
+	/*
+	 * The caller uses "no_file_match" if it wants to match specific devs
+	 * itself, instead of matching everything in device_ids_match.
+	 */
+	if (no_file_match && cmd->enable_devices_file)
+		return 1;
+
 	/*
 	 * Match entries from cmd->use_devices with device structs in dev-cache.
 	 */
@@ -1987,6 +2032,16 @@ int setup_devices(struct cmd_context *cmd)
 	return 1;
 }
 
+int setup_devices(struct cmd_context *cmd)
+{
+	return _setup_devices(cmd, 0);
+}
+
+int setup_devices_no_file_match(struct cmd_context *cmd)
+{
+	return _setup_devices(cmd, 1);
+}
+
 /*
  * The alternative to setup_devices() when the command is interested
  * in using only one PV.
@@ -2055,188 +2110,3 @@ int setup_device(struct cmd_context *cmd, const char *devname)
 	return 1;
 }
 
-/*
- * pvscan --cache is specialized/optimized to look only at command args,
- * so this just sets up the devices file, then individual devices are
- * added to dev-cache and matched with device_ids later in pvscan.
- */
-
-int setup_devices_for_pvscan_cache(struct cmd_context *cmd)
-{
-	if (cmd->enable_devices_list) {
-		if (!_setup_devices_list(cmd))
-			return_0;
-		return 1;
-	}
-
-	if (!setup_devices_file(cmd))
-		return_0;
-
-	if (!cmd->enable_devices_file)
-		return 1;
-
-	if (!devices_file_exists(cmd)) {
-		log_debug("Devices file not found, ignoring.");
-		cmd->enable_devices_file = 0;
-		return 1;
-	}
-
-	if (!lock_devices_file(cmd, LOCK_SH)) {
-		log_error("Failed to lock the devices file to read.");
-		return 0;
-	}
-
-	if (!device_ids_read(cmd)) {
-		log_error("Failed to read the devices file.");
-		unlock_devices_file(cmd);
-		return 0;
-	}
-
-	unlock_devices_file(cmd);
-	return 1;
-}
-
-
-/* Get a device name from a devno. */
-
-static char *_get_devname_from_devno(struct cmd_context *cmd, dev_t devno)
-{
-	char path[PATH_MAX];
-	char devname[PATH_MAX];
-	char namebuf[NAME_LEN];
-	char line[1024];
-	int major = MAJOR(devno);
-	int minor = MINOR(devno);
-	int line_major;
-	int line_minor;
-	uint64_t line_blocks;
-	DIR *dir;
-	struct dirent *dirent;
-	FILE *fp;
-
-	/*
-	 * $ ls /sys/dev/block/8:0/device/block/
-	 * sda
-	 */
-	if (major_is_scsi_device(cmd->dev_types, major)) {
-		if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/device/block",
-				dm_sysfs_dir(), major, minor) < 0) {
-			return NULL;
-		}
-
-		if (!(dir = opendir(path)))
-			return NULL;
-
-		while ((dirent = readdir(dir))) {
-			if (dirent->d_name[0] == '.')
-				continue;
-			if (dm_snprintf(devname, sizeof(devname), "/dev/%s", dirent->d_name) < 0) {
-				devname[0] = '\0';
-				stack;
-			}
-			break;
-		}
-		closedir(dir);
-
-		if (devname[0]) {
-			log_debug("Found %s for %d:%d from sys", devname, major, minor);
-			return _strdup(devname);
-		}
-		return NULL;
-	}
-
-	/*
-	 * $ cat /sys/dev/block/253:3/dm/name
-	 * mpatha
-	 */
-	if (major == cmd->dev_types->device_mapper_major) {
-		if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/dm/name",
-				dm_sysfs_dir(), major, minor) < 0) {
-			return NULL;
-		}
-
-		if (!get_sysfs_value(path, namebuf, sizeof(namebuf), 0))
-			return NULL;
-
-		if (dm_snprintf(devname, sizeof(devname), "/dev/mapper/%s", namebuf) < 0) {
-			devname[0] = '\0';
-			stack;
-		}
-
-		if (devname[0]) {
-			log_debug("Found %s for %d:%d from sys", devname, major, minor);
-			return _strdup(devname);
-		}
-		return NULL;
-	}
-
-	/*
-	 * /proc/partitions lists
-	 * major minor #blocks name
-	 */
-
-	if (!(fp = fopen("/proc/partitions", "r")))
-		return NULL;
-
-	while (fgets(line, sizeof(line), fp)) {
-		if (sscanf(line, "%u %u %llu %s", &line_major, &line_minor, (unsigned long long *)&line_blocks, namebuf) != 4)
-			continue;
-		if (line_major != major)
-			continue;
-		if (line_minor != minor)
-			continue;
-
-		if (dm_snprintf(devname, sizeof(devname), "/dev/%s", namebuf) < 0) {
-			devname[0] = '\0';
-			stack;
-		}
-		break;
-	}
-	fclose(fp);
-
-	if (devname[0]) {
-		log_debug("Found %s for %d:%d from proc", devname, major, minor);
-		return _strdup(devname);
-	}
-
-	/*
-	 * If necessary, this could continue searching by stat'ing /dev entries.
-	 */
-
-	return NULL;
-}
-
-int setup_devname_in_dev_cache(struct cmd_context *cmd, const char *devname)
-{
-	struct stat buf;
-	struct device *dev;
-
-	if (stat(devname, &buf) < 0) {
-		log_error("Cannot access device %s.", devname);
-		return 0;
-	}
-
-	if (!S_ISBLK(buf.st_mode)) {
-		log_error("Invaild device type %s.", devname);
-		return 0;
-	}
-
-	if (!_insert_dev(devname, buf.st_rdev))
-		return_0;
-
-	if (!(dev = (struct device *) dm_hash_lookup(_cache.names, devname)))
-		return_0;
-
-	return 1;
-}
-
-int setup_devno_in_dev_cache(struct cmd_context *cmd, dev_t devno)
-{
-	const char *devname;
-
-	if (!(devname = _get_devname_from_devno(cmd, devno)))
-		return_0;
-
-	return setup_devname_in_dev_cache(cmd, devname);
-}
-

lib/device/dev-cache.h

@@ -53,8 +53,9 @@ int dev_cache_has_scanned(void);
 
 int dev_cache_add_dir(const char *path);
 struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f);
-
-struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t device, struct dev_filter *f, int *filtered);
+struct device *dev_cache_get_existing(struct cmd_context *cmd, const char *name, struct dev_filter *f);
+struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t devt);
+void dev_cache_verify_aliases(struct device *dev);
 
 struct device *dev_hash_get(const char *name);
 
@@ -73,15 +74,12 @@ void dev_cache_failed_path(struct device *dev, const char *path);
 bool dev_cache_has_md_with_end_superblock(struct dev_types *dt);
 
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value);
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen);
 int get_dm_uuid_from_sysfs(char *buf, size_t buf_size, int major, int minor);
 
 int setup_devices_file(struct cmd_context *cmd);
 int setup_devices(struct cmd_context *cmd);
+int setup_devices_no_file_match(struct cmd_context *cmd);
 int setup_device(struct cmd_context *cmd, const char *devname);
 
-/* Normal device setup functions are split up for pvscan optimization. */
-int setup_devices_for_pvscan_cache(struct cmd_context *cmd);
-int setup_devname_in_dev_cache(struct cmd_context *cmd, const char *devname);
-int setup_devno_in_dev_cache(struct cmd_context *cmd, dev_t devno);
-
 #endif

lib/device/dev-io.c

@@ -58,6 +58,9 @@ static int _dev_get_size_file(struct device *dev, uint64_t *size)
 	const char *name = dev_name(dev);
 	struct stat info;
 
+	if (dm_list_empty(&dev->aliases))
+		return_0;
+
 	if (dev->size_seqno == _dev_size_seqno) {
 		log_very_verbose("%s: using cached size %" PRIu64 " sectors",
 				 name, dev->size);
@@ -87,7 +90,7 @@ static int _dev_get_size_dev(struct device *dev, uint64_t *size)
 	int do_close = 0;
 
 	if (dm_list_empty(&dev->aliases))
-		return 0;
+		return_0;
 
 	if (dev->size_seqno == _dev_size_seqno) {
 		log_very_verbose("%s: using cached size %" PRIu64 " sectors",
@@ -305,6 +308,13 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 	if ((flags & O_EXCL))
 		need_excl = 1;
 
+	if (dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_print("Cannot open device %d:%d with no valid paths.", (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		return 0;
+	}
+	name = dev_name(dev);
+
 	if (dev->fd >= 0) {
 		if (((dev->flags & DEV_OPENED_RW) || !need_rw) &&
 		    ((dev->flags & DEV_OPENED_EXCL) || !need_excl)) {
@@ -314,7 +324,7 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 
 		if (dev->open_count && !need_excl)
 			log_debug_devs("%s: Already opened read-only. Upgrading "
-				       "to read-write.", dev_name(dev));
+				       "to read-write.", name);
 
 		/* dev_close_immediate will decrement this */
 		dev->open_count++;
@@ -327,11 +337,7 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 
 	if (critical_section())
 		/* FIXME Make this log_error */
-		log_verbose("dev_open(%s) called while suspended",
-			    dev_name(dev));
-
-	if (!(name = dev_name_confirmed(dev, quiet)))
-		return_0;
+		log_verbose("dev_open(%s) called while suspended", name);
 
 #ifdef O_DIRECT_SUPPORT
 	if (direct) {
@@ -372,9 +378,9 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 		}
 #endif
 		if (quiet)
-			log_sys_debug("open", name);
+			log_debug("Failed to open device path %s (%d).", name, errno);
 		else
-			log_sys_error("open", name);
+			log_error("Failed to open device path %s (%d).", name, errno);
 
 		dev->flags |= DEV_OPEN_FAILURE;
 		return 0;
@@ -415,10 +421,12 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 	if ((flags & O_CREAT) && !(flags & O_TRUNC))
 		dev->end = lseek(dev->fd, (off_t) 0, SEEK_END);
 
-	log_debug_devs("Opened %s %s%s%s", dev_name(dev),
-		       dev->flags & DEV_OPENED_RW ? "RW" : "RO",
-		       dev->flags & DEV_OPENED_EXCL ? " O_EXCL" : "",
-		       dev->flags & DEV_O_DIRECT ? " O_DIRECT" : "");
+	if (!quiet) {
+		log_debug_devs("Opened %s %s%s%s", name,
+				dev->flags & DEV_OPENED_RW ? "RW" : "RO",
+				dev->flags & DEV_OPENED_EXCL ? " O_EXCL" : "",
+				dev->flags & DEV_O_DIRECT ? " O_DIRECT" : "");
+	}
 
 	dev->flags &= ~DEV_OPEN_FAILURE;
 	return 1;

lib/device/dev-mpath.c

@@ -17,12 +17,14 @@
 #include "lib/activate/activate.h"
 #include "lib/commands/toolcontext.h"
 #include "lib/device/device_id.h"
+#include "lib/datastruct/str_list.h"
 #ifdef UDEV_SYNC_SUPPORT
 #include <libudev.h>
 #include "lib/device/dev-ext-udev-constants.h"
 #endif
 
 #include <dirent.h>
+#include <ctype.h>
 
 #define MPATH_PREFIX "mpath-"
 
@@ -35,21 +37,175 @@
  * If dm-3 is not an mpath device, then the constant "1" is stored in
  * the hash table with the key of the dm minor number.
  */
-static struct dm_pool *_hash_mem;
+static struct dm_pool *_wwid_mem;
 static struct dm_hash_table *_minor_hash_tab;
 static struct dm_hash_table *_wwid_hash_tab;
+static struct dm_list _ignored;
+static struct dm_list _ignored_exceptions;
 
 #define MAX_WWID_LINE 512
 
-/*
- * do we need to check the multipath.conf blacklist?
- */
+static void _read_blacklist_file(const char *path)
+{
+	FILE *fp;
+	char line[MAX_WWID_LINE];
+	char wwid[MAX_WWID_LINE];
+	char *word, *p;
+	int section_black = 0;
+	int section_exceptions = 0;
+	int found_quote;
+	int found_type;
+	int i, j;
 
-static void _read_wwid_file(const char *config_wwids_file)
+	if (!(fp = fopen(path, "r")))
+		return;
+
+	while (fgets(line, sizeof(line), fp)) {
+		word = NULL;
+
+		/* skip initial white space on the line */
+		for (i = 0; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (isspace(line[i]))
+				continue;
+			word = &line[i];
+			break;
+		}
+
+		if (!word || word[0] == '#')
+			continue;
+
+		/* identify the start of the section we want to read */
+		if (strchr(word, '{')) {
+			if (!strncmp(word, "blacklist_exceptions", 20))
+				section_exceptions = 1;
+			else if (!strncmp(word, "blacklist", 9))
+				section_black = 1;
+			continue;
+		}
+		/* identify the end of the section we've been reading */
+		if (strchr(word, '}')) {
+			section_exceptions = 0;
+			section_black = 0;
+			continue;
+		}
+		/* skip lines that are not in a section we want */
+		if (!section_black && !section_exceptions)
+			continue;
+
+		/*
+		 * read a wwid from the blacklist{_exceptions} section.
+		 * does not recognize other non-wwid entries in the
+		 * section, and skips those (should the entire mp
+		 * config filtering be disabled if non-wwids are seen?
+		 */
+		if (!(p = strstr(word, "wwid")))
+			continue;
+
+		i += 4; /* skip "wwid" */
+
+		/*
+		 * copy wwid value from the line.
+		 * the wwids copied here need to match the
+		 * wwids read from /etc/multipath/wwids,
+		 * which are matched to wwids from sysfs.
+		 */
+
+		memset(wwid, 0, sizeof(wwid));
+		found_quote = 0;
+		found_type = 0;
+		j = 0;
+
+		for (; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (!j && isspace(line[i]))
+				continue;
+			if (isspace(line[i]))
+				break;
+			/* quotes around wwid are optional */
+			if ((line[i] == '"') && !found_quote) {
+				found_quote = 1;
+				continue;
+			}
+			/* second quote is end of wwid */
+			if ((line[i] == '"') && found_quote)
+				break;
+			/* exclude initial 3/2/1 for naa/eui/t10 */
+			if (!j && !found_type &&
+			    ((line[i] == '3') || (line[i] == '2') || (line[i] == '1'))) {
+				found_type = 1;
+				continue;
+			}
+
+			wwid[j] = line[i];
+			j++;
+		}
+
+		if (j < 8)
+			continue;
+
+		log_debug("multipath wwid %s in %s %s",
+			  wwid, section_exceptions ? "blacklist_exceptions" : "blacklist", path);
+
+		if (section_exceptions) {
+			if (!str_list_add(_wwid_mem, &_ignored_exceptions, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		} else {
+			if (!str_list_add(_wwid_mem, &_ignored, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		}
+	}
+
+	if (fclose(fp))
+		stack;
+}
+
+static void _read_wwid_exclusions(void)
+{
+	char path[PATH_MAX] = { 0 };
+	DIR *dir;
+	struct dirent *de;
+	struct dm_str_list *sl, *sl2;
+	int rem_count = 0;
+
+	_read_blacklist_file("/etc/multipath.conf");
+
+	if ((dir = opendir("/etc/multipath/conf.d"))) {
+		while ((de = readdir(dir))) {
+			if (de->d_name[0] == '.')
+				continue;
+			snprintf(path, PATH_MAX-1, "/etc/multipath/conf.d/%s", de->d_name);
+			_read_blacklist_file(path);
+		}
+		closedir(dir);
+	}
+
+	/* for each wwid in ignored_exceptions, remove it from ignored */
+
+	dm_list_iterate_items_safe(sl, sl2, &_ignored) {
+		if (str_list_match_item(&_ignored_exceptions, sl->str))
+			str_list_del(&_ignored, sl->str);
+	}
+
+	/* for each wwid in ignored, remove it from wwid_hash */
+
+	dm_list_iterate_items(sl, &_ignored) {
+		dm_hash_remove_binary(_wwid_hash_tab, sl->str, strlen(sl->str));
+		rem_count++;
+	}
+
+	if (rem_count)
+		log_debug("multipath config ignored %d wwids", rem_count);
+}
+
+static void _read_wwid_file(const char *config_wwids_file, int *entries)
 {
 	FILE *fp;
 	char line[MAX_WWID_LINE];
 	char *wwid, *p;
+	char typestr[2] = { 0 };
 	int count = 0;
 
 	if (config_wwids_file[0] != '/') {
@@ -71,8 +227,17 @@ static void _read_wwid_file(const char *config_wwids_file)
 		if (line[0] == '/')
 			wwid++;
 
-		/* skip the initial '3' */
-		wwid++;
+
+		/*
+		 * the initial character is the id type,
+		 * 1 is t10, 2 is eui, 3 is naa, 8 is scsi name.
+		 * wwids are stored in the hash table without the type charater.
+		 * It seems that sometimes multipath does not include
+		 * the type charater (seen with t10 scsi_debug devs).
+		 */
+		typestr[0] = *wwid;
+		if (typestr[0] == '1' || typestr[0] == '2' || typestr[0] == '3')
+			wwid++;
 
 		if ((p = strchr(wwid, '/')))
 			*p = '\0';
@@ -85,6 +250,7 @@ static void _read_wwid_file(const char *config_wwids_file)
 		stack;
 
 	log_debug("multipath wwids read %d from %s", count, config_wwids_file);
+	*entries = count;
 }
 
 int dev_mpath_init(const char *config_wwids_file)
@@ -92,6 +258,10 @@ int dev_mpath_init(const char *config_wwids_file)
 	struct dm_pool *mem;
 	struct dm_hash_table *minor_tab;
 	struct dm_hash_table *wwid_tab;
+	int entries = 0;
+
+	dm_list_init(&_ignored);
+	dm_list_init(&_ignored_exceptions);
 
 	if (!(mem = dm_pool_create("mpath", 256))) {
 		log_error("mpath pool creation failed.");
@@ -104,7 +274,7 @@ int dev_mpath_init(const char *config_wwids_file)
 		return 0;
 	}
 
-	_hash_mem = mem;
+	_wwid_mem = mem;
 	_minor_hash_tab = minor_tab;
 
 	/* multipath_wwids_file="" disables the use of the file */
@@ -116,16 +286,24 @@ int dev_mpath_init(const char *config_wwids_file)
 	if (!(wwid_tab = dm_hash_create(110))) {
 		log_error("mpath hash table creation failed.");
 		dm_hash_destroy(_minor_hash_tab);
-		dm_pool_destroy(_hash_mem);
+		dm_pool_destroy(_wwid_mem);
 		_minor_hash_tab = NULL;
-		_hash_mem = NULL;
+		_wwid_mem = NULL;
 		return 0;
 	}
 
 	_wwid_hash_tab = wwid_tab;
 
-	if (config_wwids_file)
-		_read_wwid_file(config_wwids_file);
+	if (config_wwids_file) {
+		_read_wwid_file(config_wwids_file, &entries);
+		_read_wwid_exclusions();
+	}
+
+	if (!entries) {
+		/* reading dev wwids is skipped with null wwid_hash_tab */
+		dm_hash_destroy(_wwid_hash_tab);
+		_wwid_hash_tab = NULL;
+	}
 
 	return 1;
 }
@@ -136,12 +314,12 @@ void dev_mpath_exit(void)
 		dm_hash_destroy(_minor_hash_tab);
 	if (_wwid_hash_tab)
 		dm_hash_destroy(_wwid_hash_tab);
-	if (_hash_mem)
-		dm_pool_destroy(_hash_mem);
+	if (_wwid_mem)
+		dm_pool_destroy(_wwid_mem);
 
 	_minor_hash_tab = NULL;
 	_wwid_hash_tab = NULL;
-	_hash_mem = NULL;
+	_wwid_mem = NULL;
 }
 
 
@@ -272,10 +450,10 @@ static int _dev_is_mpath_component_udev(struct device *dev)
 }
 #endif
 
-static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev)
+static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev,
+					 int primary_result, dev_t primary_dev)
 {
 	struct dev_types *dt = cmd->dev_types;
-	const char *part_name;
 	const char *name;               /* e.g. "sda" for "/dev/sda" */
 	char link_path[PATH_MAX];       /* some obscure, unpredictable sysfs path */
 	char holders_path[PATH_MAX];    /* e.g. "/sys/block/sda/holders/" */
@@ -289,25 +467,15 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	int dm_dev_major;
 	int dm_dev_minor;
 	struct stat info;
-	dev_t primary_dev;
 	int is_mpath_component = 0;
 
-	/* multipathing is only known to exist for SCSI or NVME devices */
-	if (!major_is_scsi_device(dt, dev_major) && !dev_is_nvme(dt, dev))
-		return 0;
-
-	switch (dev_get_primary_dev(dt, dev, &primary_dev)) {
+	switch (primary_result) {
 
 	case 2: /* The dev is partition. */
-		part_name = dev_name(dev); /* name of original dev for log_debug msg */
 
 		/* gets "foo" for "/dev/foo" where "/dev/foo" comes from major:minor */
 		if (!(name = _get_sysfs_name_by_devt(sysfs_dir, primary_dev, link_path, sizeof(link_path))))
 			return_0;
-
-		log_debug_devs("%s: Device is a partition, using primary "
-			       "device %s for mpath component detection",
-			       part_name, name);
 		break;
 
 	case 1: /* The dev is already a primary dev. Just continue with the dev. */
@@ -429,47 +597,189 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	return is_mpath_component;
 }
 
-static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev)
+static int _read_sys_wwid(struct cmd_context *cmd, struct device *dev,
+			  char *idbuf, int idbufsize)
 {
-	char sysbuf[PATH_MAX] = { 0 };
+	char idtmp[DEV_WWID_SIZE];
+
+	if (!read_sys_block(cmd, dev, "device/wwid", idbuf, idbufsize)) {
+		/* the wwid file is not under device for nvme devs */
+		if (!read_sys_block(cmd, dev, "wwid", idbuf, idbufsize))
+			return 0;
+	}
+	if (!idbuf[0])
+		return 0;
+
+	/* in t10 id, replace series of spaces with one _ like multipath */
+	if (!strncmp(idbuf, "t10.", 4) && strchr(idbuf, ' ')) {
+		if (idbufsize < DEV_WWID_SIZE)
+			return 0;
+		memcpy(idtmp, idbuf, DEV_WWID_SIZE);
+		memset(idbuf, 0, idbufsize);
+		format_t10_id((const unsigned char *)idtmp, DEV_WWID_SIZE, (unsigned char *)idbuf, idbufsize);
+	}
+	return 1;
+}
+
+#define VPD_SIZE 4096
+
+static int _read_sys_vpd_wwids(struct cmd_context *cmd, struct device *dev,
+			       struct dm_list *ids)
+{
+	unsigned char vpd_data[VPD_SIZE] = { 0 };
+	int vpd_datalen = 0;
+
+	if (!read_sys_block_binary(cmd, dev, "device/vpd_pg83", (char *)vpd_data, VPD_SIZE, &vpd_datalen))
+		return 0;
+	if (!vpd_datalen)
+		return 0;
+
+	/* adds dev_wwid entry to dev->wwids for each id in vpd data */
+	parse_vpd_ids(vpd_data, vpd_datalen, ids);
+	return 1;
+}
+
+void free_wwids(struct dm_list *ids)
+{
+	struct dev_wwid *dw, *safe;
+
+	dm_list_iterate_items_safe(dw, safe, ids) {
+		dm_list_del(&dw->list);
+		free(dw);
+	}
+}
+
+static int _wwid_type_num(char *id)
+{
+	if (!strncmp(id, "naa.", 4))
+		return 3;
+	else if (!strncmp(id, "eui.", 4))
+		return 2;
+	else if (!strncmp(id, "t10.", 4))
+		return 1;
+	else
+		return -1;
+}
+
+/*
+ * TODO: if each of the different wwid types (naa/eui/t10) were
+ * represented by different DEV_ID_TYPE_FOO values, and used
+ * as device_id types, then we could drop struct dev_wwid and
+ * drop dev->wwids, and just use dev->ids for each of the
+ * different wwids found in vpd_pg83.  This would also require
+ * the ability to handle both the original method of replacing
+ * every space in the id string with _ and the new/multipath
+ * format_t10_id replacing series of spaces with one _.
+ */
+struct dev_wwid *add_wwid(char *id, int id_type, struct dm_list *ids)
+{
+	struct dev_wwid *dw;
+	int len;
+
+	if (!id_type) {
+		id_type = _wwid_type_num(id);
+		if (id_type == -1)
+			log_debug("unknown wwid type %s", id);
+	}
+
+	if (!(dw = zalloc(sizeof(struct dev_wwid))))
+		return NULL;
+	len = strlen(id);
+	if (len >= DEV_WWID_SIZE)
+		len = DEV_WWID_SIZE - 1;
+	memcpy(dw->id, id, len);
+	dw->type = id_type;
+	dm_list_add(ids, &dw->list);
+	return dw;
+}
+
+/*
+ * we save ids with format: naa.<value>, eui.<value>, t10.<value>.
+ * multipath wwids file uses format: 3<value>, 2<value>, 1<value>.
+ * The values are saved in wwid_hash_tab without the type prefix.
+ */
+
+static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev,
+			     int primary_result, dev_t primary_dev)
+{
+	char idbuf[DEV_WWID_SIZE] = { 0 };
+	struct dev_wwid *dw;
 	char *wwid;
-	long look;
 
 	if (!_wwid_hash_tab)
 		return 0;
 
-	if (!read_sys_block(cmd, dev, "device/wwid", sysbuf, sizeof(sysbuf)))
-		return 0;
-
-	if (!sysbuf[0])
-		return 0;
+	/*
+	 * Check the primary device, not the partition.
+	 */
+	if (primary_result == 2) {
+		if (!(dev = dev_cache_get_by_devt(cmd, primary_dev))) {
+			log_debug("dev_is_mpath_component %s no primary dev", dev_name(dev));
+			return 0;
+		}
+	}
 
 	/*
-	 * sysfs prints wwid as <typestr>.<value>
-	 * multipath wwid uses '3'<value>
-	 * does "<typestr>." always correspond to "3"?
+	 * This function may be called multiple times for the same device, in
+	 * particular if partitioned for each partition.
 	 */
-	if (!(wwid = strchr(sysbuf, '.')))
-		return 0;
+	if (!dm_list_empty(&dev->wwids))
+		goto lookup;
 
-	/* skip the type and dot, just as '3' was skipped from wwids entry */
-	wwid++;
-	
-	look = (long) dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid));
+	/*
+	 * Get all the ids for the device from vpd_pg83 and check if any of
+	 * those are in /etc/multipath/wwids.  These ids should include the
+	 * value printed from the sysfs wwid file.
+	 */
+	_read_sys_vpd_wwids(cmd, dev, &dev->wwids);
+	if (!dm_list_empty(&dev->wwids))
+		goto lookup;
 
-	if (look) {
-		log_debug_devs("dev_is_mpath_component %s multipath wwid %s", dev_name(dev), wwid);
-		return 1;
+	/*
+	 * This will read the sysfs wwid file, nvme devices in particular have
+	 * a wwid file but not a vpd_pg83 file.
+	 */
+	if (_read_sys_wwid(cmd, dev, idbuf, sizeof(idbuf)))
+		add_wwid(idbuf, 0, &dev->wwids);
+
+ lookup:
+	dm_list_iterate_items(dw, &dev->wwids) {
+		if (dw->type == 1 || dw->type == 2 || dw->type == 3)
+			wwid = &dw->id[4];
+		else
+			wwid = dw->id;
+
+		if (dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid))) {
+			log_debug_devs("dev_is_mpath_component %s %s in wwids file", dev_name(dev), dw->id);
+			return 1;
+		}
 	}
+
 	return 0;
 }
 
 int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
 {
-	if (_dev_is_mpath_component_sysfs(cmd, dev) == 1)
+	struct dev_types *dt = cmd->dev_types;
+	int primary_result;
+	dev_t primary_dev;
+
+	/*
+	 * multipath only uses SCSI or NVME devices
+	 */
+	if (!major_is_scsi_device(dt, MAJOR(dev->dev)) && !dev_is_nvme(dt, dev))
+		return 0;
+
+	/*
+	 * primary_result 2: dev is a partition, primary_dev is the whole device
+	 * primary_result 1: dev is a whole device
+	 */
+	primary_result = dev_get_primary_dev(dt, dev, &primary_dev);
+
+	if (_dev_is_mpath_component_sysfs(cmd, dev, primary_result, primary_dev) == 1)
 		goto found;
 
-	if (_dev_in_wwid_file(cmd, dev))
+	if (_dev_in_wwid_file(cmd, dev, primary_result, primary_dev))
 		goto found;
 
 	if (external_device_info_source() == DEV_EXT_UDEV) {
@@ -477,8 +787,85 @@ int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
 			goto found;
 	}
 
+	/*
+	 * TODO: save the result of this function in dev->flags and use those
+	 * flags on repeated calls to avoid repeating the work multiple times
+	 * for the same device when there are partitions on the device.
+	 */
+
 	return 0;
 found:
 	return 1;
 }
 
+const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev)
+{
+	char slaves_path[PATH_MAX];
+	char wwid_path[PATH_MAX];
+	char sysbuf[PATH_MAX] = { 0 };
+	char *slave_name;
+	const char *wwid = NULL;
+	struct stat info;
+	DIR *dr;
+	struct dirent *de;
+
+	/* /sys/dev/block/253:7/slaves/sda/device/wwid */
+
+	if (dm_snprintf(slaves_path, sizeof(slaves_path), "%s/dev/block/%d:%d/slaves",
+			dm_sysfs_dir(), (int)MAJOR(dev->dev), (int)MINOR(dev->dev)) < 0) {
+		log_warn("Sysfs path to check mpath components is too long.");
+		return NULL;
+	}
+
+	if (stat(slaves_path, &info))
+		return NULL;
+
+	if (!S_ISDIR(info.st_mode)) {
+		log_warn("Path %s is not a directory.", slaves_path);
+		return NULL;
+	}
+
+	/* Get wwid from first component */
+
+	if (!(dr = opendir(slaves_path))) {
+		log_debug("Device %s has no slaves dir", dev_name(dev));
+		return NULL;
+	}
+
+	while ((de = readdir(dr))) {
+		if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, ".."))
+			continue;
+
+		/* slave_name "sda" */
+		slave_name = de->d_name;
+
+		/* read /sys/block/sda/device/wwid */
+
+		if (dm_snprintf(wwid_path, sizeof(wwid_path), "%s/block/%s/device/wwid",
+       				dm_sysfs_dir(), slave_name) < 0) {
+			log_warn("Failed to create sysfs wwid path for %s", slave_name);
+			continue;
+		}
+
+		get_sysfs_value(wwid_path, sysbuf, sizeof(sysbuf), 0);
+		if (!sysbuf[0])
+			continue;
+
+		if (strstr(sysbuf, "scsi_debug")) {
+			int i;
+			for (i = 0; i < strlen(sysbuf); i++) {
+				if (sysbuf[i] == ' ')
+					sysbuf[i] = '_';
+			}
+		}
+
+		if ((wwid = dm_pool_strdup(cmd->mem, sysbuf)))
+			break;
+	}
+	if (closedir(dr))
+		stack;
+
+	return wwid;
+}
+
+

lib/device/dev-type.c

@@ -966,6 +966,9 @@ static int _wipe_known_signatures_with_blkid(struct device *dev, const char *nam
 
 	/* TODO: Should we check for valid dev - _dev_is_valid(dev)? */
 
+	if (dm_list_empty(&dev->aliases))
+		goto_out;
+
 	if (!(probe = blkid_new_probe_from_filename(dev_name(dev)))) {
 		log_error("Failed to create a new blkid probe for device %s.", dev_name(dev));
 		goto out;

lib/device/dev-type.h

@@ -63,6 +63,8 @@ int dev_is_swap(struct cmd_context *cmd, struct device *dev, uint64_t *signature
 int dev_is_luks(struct cmd_context *cmd, struct device *dev, uint64_t *signature, int full);
 int dasd_is_cdl_formatted(struct device *dev);
 
+const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev);
+
 int dev_is_lvm1(struct device *dev, char *buf, int buflen);
 int dev_is_pool(struct device *dev, char *buf, int buflen);
 

lib/device/device.h

@@ -40,6 +40,7 @@
 #define DEV_IS_NVME		0x00040000	/* set if dev is nvme */
 #define DEV_MATCHED_USE_ID	0x00080000	/* matched an entry from cmd->use_devices */
 #define DEV_SCAN_FOUND_NOLABEL	0x00100000	/* label_scan read, passed filters, but no lvm label */
+#define DEV_SCAN_NOT_READ	0x00200000	/* label_scan not able to read dev */
 
 /*
  * Support for external device info.
@@ -58,6 +59,14 @@ struct dev_ext {
 	void *handle;
 };
 
+#define DEV_WWID_SIZE 128
+
+struct dev_wwid {
+	struct dm_list list;
+	int type;
+	char id[DEV_WWID_SIZE];
+};
+
 #define DEV_ID_TYPE_SYS_WWID   0x0001
 #define DEV_ID_TYPE_SYS_SERIAL 0x0002
 #define DEV_ID_TYPE_MPATH_UUID 0x0003
@@ -104,6 +113,7 @@ struct dev_use {
  */
 struct device {
 	struct dm_list aliases;	/* struct dm_str_list */
+	struct dm_list wwids; /* struct dev_wwid, used for multipath component detection */
 	struct dm_list ids; /* struct dev_id, different entries for different idtypes */
 	struct dev_id *id; /* points to the the ids entry being used for this dev */
 	dev_t dev;
@@ -203,10 +213,11 @@ struct device *dev_create_file(const char *filename, struct device *dev,
 			       struct dm_str_list *alias, int use_malloc);
 void dev_destroy_file(struct device *dev);
 
-/* Return a valid device name from the alias list; NULL otherwise */
-const char *dev_name_confirmed(struct device *dev, int quiet);
-
 int dev_mpath_init(const char *config_wwids_file);
 void dev_mpath_exit(void);
+struct dev_wwid *add_wwid(char *id, int id_type, struct dm_list *ids);
+void free_wwids(struct dm_list *ids);
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids);
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes);
 
 #endif

lib/device/device_id.c

@@ -74,6 +74,8 @@ void unlink_searched_devnames(struct cmd_context *cmd)
 
 	if (unlink(_searched_file))
 		log_debug("unlink %s errno %d", _searched_file, errno);
+	else
+		log_debug("unlink %s", _searched_file);
 }
 
 static int _searched_devnames_exists(struct cmd_context *cmd)
@@ -180,7 +182,9 @@ void free_dids(struct dm_list *ids)
 	}
 }
 
-int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suffix, char *sysbuf, int sysbufsize)
+static int _read_sys_block(struct cmd_context *cmd, struct device *dev,
+			   const char *suffix, char *sysbuf, int sysbufsize,
+			   int binary, int *retlen)
 {
 	char path[PATH_MAX];
 	dev_t devt = dev->dev;
@@ -194,11 +198,17 @@ int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suff
 		return 0;
 	}
 
-	get_sysfs_value(path, sysbuf, sysbufsize, 0);
+	if (binary) {
+		ret = get_sysfs_binary(path, sysbuf, sysbufsize, retlen);
+		if (ret && !*retlen)
+			ret = 0;
+	} else {
+		ret = get_sysfs_value(path, sysbuf, sysbufsize, 0);
+		if (ret && !sysbuf[0])
+			ret = 0;
+	}
 
-	if (sysbuf[0]) {
-		if (prim)
-			log_debug("Using primary device_id for partition %s.", dev_name(dev));
+	if (ret) {
 		sysbuf[sysbufsize - 1] = '\0';
 		return 1;
 	}
@@ -218,6 +228,19 @@ int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suff
 	return 0;
 }
 
+int read_sys_block(struct cmd_context *cmd, struct device *dev,
+		   const char *suffix, char *sysbuf, int sysbufsize)
+{
+	return _read_sys_block(cmd, dev, suffix, sysbuf, sysbufsize, 0, NULL);
+}
+
+int read_sys_block_binary(struct cmd_context *cmd, struct device *dev,
+			  const char *suffix, char *sysbuf, int sysbufsize,
+			  int *retlen)
+{
+	return _read_sys_block(cmd, dev, suffix, sysbuf, sysbufsize, 1, retlen);
+}
+
 static int _dm_uuid_has_prefix(char *sysbuf, const char *prefix)
 {
 	if (!strncmp(sysbuf, prefix, strlen(prefix)))
@@ -241,7 +264,7 @@ static int _dm_uuid_has_prefix(char *sysbuf, const char *prefix)
 }
 
 /* the dm uuid uses the wwid of the underlying dev */
-static int _dev_has_mpath_uuid(struct cmd_context *cmd, struct device *dev, const char **idname_out)
+int dev_has_mpath_uuid(struct cmd_context *cmd, struct device *dev, const char **idname_out)
 {
 	char sysbuf[PATH_MAX] = { 0 };
 	const char *idname;
@@ -302,6 +325,7 @@ const char *device_id_system_read(struct cmd_context *cmd, struct device *dev, u
 {
 	char sysbuf[PATH_MAX] = { 0 };
 	const char *idname = NULL;
+	int i;
 
 	if (idtype == DEV_ID_TYPE_SYS_WWID) {
 		read_sys_block(cmd, dev, "device/wwid", sysbuf, sizeof(sysbuf));
@@ -309,10 +333,6 @@ const char *device_id_system_read(struct cmd_context *cmd, struct device *dev, u
 		if (!sysbuf[0])
 			read_sys_block(cmd, dev, "wwid", sysbuf, sizeof(sysbuf));
 
-		/* scsi_debug wwid begins "t10.Linux   scsi_debug ..." */
-		if (strstr(sysbuf, "scsi_debug"))
-			sysbuf[0] = '\0';
-
 		/* qemu wwid begins "t10.ATA     QEMU HARDDISK ..." */
 		if (strstr(sysbuf, "QEMU HARDDISK"))
 			sysbuf[0] = '\0';
@@ -348,11 +368,18 @@ const char *device_id_system_read(struct cmd_context *cmd, struct device *dev, u
 	}
 
 	else if (idtype == DEV_ID_TYPE_DEVNAME) {
+		if (dm_list_empty(&dev->aliases))
+			goto_bad;
 		if (!(idname = strdup(dev_name(dev))))
 			goto_bad;
 		return idname;
 	}
 
+	for (i = 0; i < strlen(sysbuf); i++) {
+		if (isblank(sysbuf[i]) || isspace(sysbuf[i]) || iscntrl(sysbuf[i]))
+			sysbuf[i] = '_';
+	}
+
 	if (!sysbuf[0])
 		goto_bad;
 
@@ -671,6 +698,9 @@ int device_ids_write(struct cmd_context *cmd)
 		cmd->enable_devices_file = 1;
 	}
 
+	if (test_mode())
+		return 1;
+
 	if (_devices_file_version[0]) {
 		if (sscanf(_devices_file_version, "%u.%u.%u", &df_major, &df_minor, &df_counter) != 3) {
 			/* don't update a file we can't parse */
@@ -780,7 +810,7 @@ static void _device_ids_update_try(struct cmd_context *cmd)
 
 	/* Defer updates to non-pvscan-cache commands. */
 	if (cmd->pvscan_cache_single) {
-		log_print("pvscan[%d] skip updating devices file.", getpid());
+		log_print("Devices file update skipped."); 
 		return;
 	}
 
@@ -887,7 +917,7 @@ struct dev_use *get_du_for_pvid(struct cmd_context *cmd, const char *pvid)
 	return NULL;
 }
 
-static struct dev_use *_get_du_for_devname(struct cmd_context *cmd, const char *devname)
+struct dev_use *get_du_for_devname(struct cmd_context *cmd, const char *devname)
 {
 	struct dev_use *du;
 
@@ -928,6 +958,14 @@ int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid_
 	struct dev_use *du, *update_du = NULL, *du_dev, *du_pvid, *du_devname, *du_devid;
 	struct dev_id *id;
 	int found_id = 0;
+	int part = 0;
+
+	if (!dev_get_partition_number(dev, &part))
+		return_0;
+
+	/* Ensure valid dev_name(dev) below. */
+	if (dm_list_empty(&dev->aliases))
+		return_0;
 
 	/*
 	 * When enable_devices_file=0 and pending_devices_file=1 we let
@@ -946,10 +984,6 @@ int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid_
 	 */
 	memcpy(&pvid, pvid_arg, ID_LEN);
 
-	du_dev = get_du_for_dev(cmd, dev);
-	du_pvid = get_du_for_pvid(cmd, pvid);
-	du_devname = _get_du_for_devname(cmd, dev_name(dev));
-
 	/*
 	 * Choose the device_id type for the device being added.
 	 *
@@ -981,7 +1015,7 @@ int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid_
 	}
 
 	if (MAJOR(dev->dev) == cmd->dev_types->device_mapper_major) {
-		if (_dev_has_mpath_uuid(cmd, dev, &idname)) {
+		if (dev_has_mpath_uuid(cmd, dev, &idname)) {
 			idtype = DEV_ID_TYPE_MPATH_UUID;
 			goto id_done;
 		}
@@ -1065,6 +1099,9 @@ id_done:
 	idtype = 0;
 
 	/*
+	 * "dev" is the device we are adding.
+	 * "id" is the device_id it's using, set in dev->id.
+	 *
 	 * Update the cmd->use_devices list for the new device.  The
 	 * use_devices list will be used to update the devices file.
 	 *
@@ -1076,23 +1113,58 @@ id_done:
 	 * those other entries to fix any incorrect info.
 	 */
 
+	/* Is there already an entry matched to this device? */
+	du_dev = get_du_for_dev(cmd, dev);
+
+	/* Is there already an entry matched to this device's pvid? */
+	du_pvid = get_du_for_pvid(cmd, pvid);
+
+	/* Is there already an entry using this device's name? */
+	du_devname = get_du_for_devname(cmd, dev_name(dev));
+
+	/* Is there already an entry using the device_id for this device? */
 	du_devid = _get_du_for_device_id(cmd, id->idtype, id->idname);
 
 	if (du_dev)
-		log_debug("device_id_add %s pvid %s matches du_dev %p dev %s",
+		log_debug("device_id_add %s pvid %s matches entry %p dev %s",
 			  dev_name(dev), pvid, du_dev, dev_name(du_dev->dev));
 	if (du_pvid)
-		log_debug("device_id_add %s pvid %s matches du_pvid %p dev %s pvid %s",
+		log_debug("device_id_add %s pvid %s matches entry %p dev %s with same pvid %s",
 			  dev_name(dev), pvid, du_pvid, du_pvid->dev ? dev_name(du_pvid->dev) : ".",
 			  du_pvid->pvid);
 	if (du_devid)
-		log_debug("device_id_add %s pvid %s matches du_devid %p dev %s pvid %s",
+		log_debug("device_id_add %s pvid %s matches entry %p dev %s with same device_id %d %s",
 			  dev_name(dev), pvid, du_devid, du_devid->dev ? dev_name(du_devid->dev) : ".",
-			  du_devid->pvid);
+			  du_devid->idtype, du_devid->idname);
 	if (du_devname)
-		log_debug("device_id_add %s pvid %s matches du_devname %p dev %s pvid %s",
+		log_debug("device_id_add %s pvid %s matches entry %p dev %s with same devname %s",
 			  dev_name(dev), pvid, du_devname, du_devname->dev ? dev_name(du_devname->dev) : ".",
-			  du_devname->pvid);
+			  du_devname->devname);
+
+	if (du_pvid && (du_pvid->dev != dev))
+		log_warn("WARNING: adding device %s with PVID %s which is already used for %s device_id %s.",
+			 dev_name(dev), pvid, du_pvid->dev ? dev_name(du_pvid->dev) : "missing device",
+			 du_pvid->idname ?: "none");
+
+	if (du_devid && (du_devid->dev != dev)) {
+		if (!du_devid->dev) {
+			log_warn("WARNING: adding device %s with idname %s which is already used for missing device.",
+				 dev_name(dev), id->idname);
+		} else {
+			int ret1, ret2;
+			dev_t devt1, devt2;
+			/* Check if both entries are partitions of the same device. */
+			ret1 = dev_get_primary_dev(cmd->dev_types, dev, &devt1);
+			ret2 = dev_get_primary_dev(cmd->dev_types, du_devid->dev, &devt2);
+			if ((ret1 == 2) && (ret2 == 2) && (devt1 == devt2)) {
+				log_debug("Using separate entries for partitions of same device %s part %d %s part %d.",
+					  dev_name(dev), part, dev_name(du_devid->dev), du_devid->part);
+			} else {
+				log_warn("WARNING: adding device %s with idname %s which is already used for %s.",
+					 dev_name(dev), id->idname, dev_name(du_devid->dev));
+			}
+		}
+	}
 
 	/*
 	 * If one of the existing entries (du_dev, du_pvid, du_devid, du_devname)
@@ -1105,29 +1177,6 @@ id_done:
 		dm_list_del(&update_du->list);
 		update_matching_kind = "device";
 		update_matching_name = dev_name(dev);
-
-		if (du_devid && (du_devid != du_dev)) {
-			log_warn("WARNING: device %s (%s) and %s (%s) have duplicate device ID.",
-				 dev_name(dev), id->idname,
-				 (du_pvid && du_pvid->dev) ? dev_name(du_pvid->dev) : "none",
-				 du_pvid ? du_pvid->idname : "");
-		}
-
-		if (du_pvid && (du_pvid != du_dev)) {
-			log_warn("WARNING: device %s (%s) and %s (%s) have duplicate PVID %s",
-				 dev_name(dev), id->idname,
-				 du_pvid->dev ? dev_name(du_pvid->dev) : "none", du_pvid->idname,
-				 pvid);
-		}
-
-		if (du_devname && (du_devname != du_dev)) {
-			/* clear devname in another entry with our devname */
-			log_warn("Devices file PVID %s clearing wrong DEVNAME %s.",
-				 du_devname->pvid, du_devname->devname);
-			free(du_devname->devname);
-			du_devname->devname = NULL;
-		}
-
 	} else if (du_pvid) {
 		/*
 		 * If the device_id of the existing entry for PVID is the same
@@ -1141,17 +1190,12 @@ id_done:
 		else
 			check_idname = device_id_system_read(cmd, dev, du_pvid->idtype);
 
-		if (check_idname && !strcmp(check_idname, du_pvid->idname)) {
+		if (!du_pvid->idname || (check_idname && !strcmp(check_idname, du_pvid->idname))) {
 			update_du = du_pvid;
 			dm_list_del(&update_du->list);
 			update_matching_kind = "PVID";
 			update_matching_name = pvid;
 		} else {
-			log_warn("WARNING: device %s (%s) and %s (%s) have duplicate PVID %s",
-				 dev_name(dev), id->idname,
-				 du_pvid->dev ? dev_name(du_pvid->dev) : "none", du_pvid->idname,
-				 pvid);
-
 			if (!cmd->current_settings.yes &&
 			    yes_no_prompt("Add device with duplicate PV to devices file?") == 'n') {
 				log_print("Device not added.");
@@ -1159,21 +1203,6 @@ id_done:
 				return 1;
 			}
 		}
-
-		if (du_devid && (du_devid != du_pvid)) {
-			/* warn about another entry using the same device_id */
-			log_warn("WARNING: duplicate device_id %s for PVIDs %s %s",
-				 du_devid->idname, du_devid->pvid, du_pvid->pvid);
-		}
-
-		if (du_devname && (du_devname != du_pvid)) {
-			/* clear devname in another entry with our devname */
-			log_warn("Devices file PVID %s clearing wrong DEVNAME %s.",
-				 du_devname->pvid, du_devname->devname);
-			free(du_devname->devname);
-			du_devname->devname = NULL;
-		}
-
 	} else if (du_devid) {
 		/*
 		 * Do we create a new du or update the existing du?
@@ -1188,64 +1217,13 @@ id_done:
 		 * the same device_id (create a new du for dev.)
 		 * If not, then update the existing du_devid.
 		 */
-		
-		if (du_devid->dev != dev)
-			check_idname = device_id_system_read(cmd, du_devid->dev, id->idtype);
-
-		if (check_idname && !strcmp(check_idname, id->idname)) {
-			int ret1, ret2;
-			dev_t devt1, devt2;
-
-			/*
-			 * two different devices have the same device_id,
-			 * create a new du for the device being added
-			 */
-
-			/* dev_is_partitioned() the dev open to read it. */
-			if (!label_scan_open(du_devid->dev))
-				log_warn("Cannot open %s", dev_name(du_devid->dev));
-
-			if (dev_is_partitioned(cmd, du_devid->dev)) {
-				/* Check if existing entry is whole device and new entry is a partition of it. */
-				ret1 = dev_get_primary_dev(cmd->dev_types, dev, &devt1);
-				if ((ret1 == 2) && (devt1 == du_devid->dev->dev))
-					log_warn("Remove partitioned device %s from devices file.", dev_name(du_devid->dev));
-			} else {
-				/* Check if both entries are partitions of the same device. */
-				ret1 = dev_get_primary_dev(cmd->dev_types, dev, &devt1);
-				ret2 = dev_get_primary_dev(cmd->dev_types, du_devid->dev, &devt2);
-
-				if ((ret1 == 2) && (ret2 == 2) && (devt1 == devt2)) {
-					log_warn("Partitions %s %s have same device_id %s",
-						 dev_name(dev), dev_name(du_devid->dev), id->idname);
-				} else {
-					log_warn("Duplicate device_id %s %s for %s and %s",
-						 idtype_to_str(id->idtype), check_idname,
-						 dev_name(dev), dev_name(du_devid->dev));
-				}
-			}
-		} else {
+		if (du_devid->dev == dev) {
 			/* update the existing entry with matching devid */
 			update_du = du_devid;
 			dm_list_del(&update_du->list);
 			update_matching_kind = "device_id";
 			update_matching_name = id->idname;
 		}
-
-		if (du_devname && (du_devname != du_devid)) {
-			/* clear devname in another entry with our devname */
-			log_warn("Devices file PVID %s clearing wrong DEVNAME %s",
-				 du_devname->pvid, du_devname->devname);
-			free(du_devname->devname);
-			du_devname->devname = NULL;
-		}
-
-	} else if (du_devname) {
-		/* clear devname in another entry with our devname */
-		log_warn("Devices file PVID %s clearing wrong DEVNAME %s",
-			 du_devname->pvid, du_devname->devname);
-		free(du_devname->devname);
-		du_devname->devname = NULL;
 	}
 
 	free((void *)check_idname);
@@ -1316,15 +1294,15 @@ void device_id_update_vg_uuid(struct cmd_context *cmd, struct volume_group *vg,
 	int update = 0;
 
 	if (!cmd->enable_devices_file)
-		goto out;
+		return;
 
 	/* Without this setting there is no stacking LVs on PVs. */
 	if (!cmd->scan_lvs)
-		goto out;
+		return;
 
 	/* Check if any devices file entries are stacked on LVs. */
 	if (!_device_ids_use_lvmlv(cmd))
-		goto out;
+		return;
 
 	memcpy(old_vgid, old_vg_id, ID_LEN);
 	memcpy(new_vgid, &vg->id, ID_LEN);
@@ -1354,12 +1332,15 @@ void device_id_update_vg_uuid(struct cmd_context *cmd, struct volume_group *vg,
 	if (update &&
 	    !device_ids_write(cmd))
 		stack;
- out:
 	unlock_devices_file(cmd);
 }
 
 static int _idtype_compatible_with_major_number(struct cmd_context *cmd, int idtype, int major)
 {
+	/* devname can be used with any kind of device */
+	if (idtype == DEV_ID_TYPE_DEVNAME)
+		return 1;
+
 	if (idtype == DEV_ID_TYPE_MPATH_UUID ||
 	    idtype == DEV_ID_TYPE_CRYPT_UUID ||
 	    idtype == DEV_ID_TYPE_LVMLV_UUID)
@@ -1388,6 +1369,43 @@ static int _idtype_compatible_with_major_number(struct cmd_context *cmd, int idt
 	return 1;
 }
 
+static int _match_dm_devnames(struct cmd_context *cmd, struct device *dev,
+			      struct dev_id *id, struct dev_use *du)
+{
+	struct stat buf;
+
+	if (MAJOR(dev->dev) != cmd->dev_types->device_mapper_major)
+		return 0;
+
+	if (id->idname && du->idname && !strcmp(id->idname, du->idname))
+		return 1;
+
+	if (du->idname && !strcmp(du->idname, dev_name(dev))) {
+		log_debug("Match device_id %s %s to %s: ignoring idname %s",
+			  idtype_to_str(du->idtype), du->idname, dev_name(dev), id->idname ?: ".");
+		return 1;
+	}
+
+	if (!du->idname)
+		return 0;
+
+	/* detect that a du entry is for a dm device */
+
+	if (!strncmp(du->idname, "/dev/dm-", 8) || !strncmp(du->idname, "/dev/mapper/", 12)) {
+		if (stat(du->idname, &buf))
+			return 0;
+
+		if ((MAJOR(buf.st_rdev) == cmd->dev_types->device_mapper_major) &&
+		    (MINOR(buf.st_rdev) == MINOR(dev->dev))) {
+			log_debug("Match device_id %s %s to %s: using other dm name, ignoring %s",
+				  idtype_to_str(du->idtype), du->idname, dev_name(dev), id->idname ?: ".");
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
 /*
  * check for dev->ids entry with du->idtype, if found compare it,
  * if not, system_read of this type and add entry to dev->ids, compare it.
@@ -1400,43 +1418,74 @@ static int _match_du_to_dev(struct cmd_context *cmd, struct dev_use *du, struct
 	const char *idname;
 	int part;
 
-	if (!du->idname || !du->idtype)
+	/*
+	 * The idname will be removed from an entry with devname type when the
+	 * devname is read and found to hold a different PVID than the PVID in
+	 * the entry.  At that point we only have the PVID and no known
+	 * location for it.
+	 */
+	if (!du->idname || !du->idtype) {
+		/*
+		log_debug("Mismatch device_id %s %s %s to %s",
+			  du->idtype ? idtype_to_str(du->idtype) : "idtype_missing",
+			  du->idname ? du->idname : "idname_missing",
+			  du->devname ? du->devname : "devname_missing",
+			  dev_name(dev));
+		*/
 		return 0;
+	}
 
 	/*
 	 * Some idtypes can only match devices with a specific major number,
 	 * so we can skip trying to match certain du entries based simply on
 	 * the major number of dev.
 	 */
-	if (!_idtype_compatible_with_major_number(cmd, du->idtype, (int)MAJOR(dev->dev)))
+	if (!_idtype_compatible_with_major_number(cmd, du->idtype, (int)MAJOR(dev->dev))) {
+		/*
+		log_debug("Mismatch device_id %s %s to %s: wrong major",
+			  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev));
+		*/
 		return 0;
+	}
 
 	if (!dev_get_partition_number(dev, &part)) {
-		log_debug("compare %s failed to get dev partition", dev_name(dev));
+		/*
+		log_debug("Mismatch device_id %s %s to %s: no partition",
+			  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev));
+		*/
 		return 0;
 	}
 	if (part != du->part) {
 		/*
-		log_debug("compare mis %s %s part %d to %s part %d",
-			  idtype_to_str(du->idtype), du->idname ?: ".", du->part, dev_name(dev), part);
+		log_debug("Mismatch device_id %s %s to %s: wrong partition %d vs %d",
+			  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev), du->part, part);
 		*/
 		return 0;
 	}
 
 	dm_list_iterate_items(id, &dev->ids) {
 		if (id->idtype == du->idtype) {
-			if (id->idname && !strcmp(id->idname, du->idname)) {
+			if ((id->idtype == DEV_ID_TYPE_DEVNAME) && _match_dm_devnames(cmd, dev, id, du)) {
+				/* dm devs can have differing names that we know still match */
+				du->dev = dev;
+				dev->id = id;
+				dev->flags |= DEV_MATCHED_USE_ID;
+				log_debug("Match device_id %s %s to %s: dm names",
+					  idtype_to_str(du->idtype), du->idname, dev_name(dev));
+				return 1;
+
+			} else if (id->idname && !strcmp(id->idname, du->idname)) {
 				du->dev = dev;
 				dev->id = id;
 				dev->flags |= DEV_MATCHED_USE_ID;
 				log_debug("Match device_id %s %s to %s",
 					  idtype_to_str(du->idtype), du->idname, dev_name(dev));
 				return 1;
+
 			} else {
 				/*
-				log_debug("compare mis %s %s to %s %s",
-			  		  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev),
-					  ((id->idtype != DEV_ID_TYPE_DEVNAME) && id->idname) ? id->idname : "");
+				log_debug("Mismatch device_id %s %s to %s: idname %s",
+			  		  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev), id->idname ?: ":");
 				*/
 				return 0;
 			}
@@ -1456,7 +1505,7 @@ static int _match_du_to_dev(struct cmd_context *cmd, struct dev_use *du, struct
 		id->dev = dev;
 		dm_list_add(&dev->ids, &id->list);
 		/*
-		log_debug("compare mis %s %s to %s no idtype",
+		log_debug("Mismatch device_id %s %s to %s: no idtype",
 			  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev));
 		*/
 		return 0;
@@ -1481,9 +1530,8 @@ static int _match_du_to_dev(struct cmd_context *cmd, struct dev_use *du, struct
 	}
 
 	/*
-	log_debug("compare mis %s %s to %s %s",
-		  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev),
-		  ((id->idtype != DEV_ID_TYPE_DEVNAME) && id->idname) ? id->idname : "");
+	log_debug("Mismatch device_id %s %s to %s: idname %s",
+		  idtype_to_str(du->idtype), du->idname ?: ".", dev_name(dev), idname);
 	*/
 	return 0;
 }
@@ -1493,7 +1541,7 @@ int device_ids_match_dev(struct cmd_context *cmd, struct device *dev)
 	struct dev_use *du;
 
 	/* First check the du entry with matching devname since it's likely correct. */
-	if ((du = _get_du_for_devname(cmd, dev_name(dev)))) {
+	if ((du = get_du_for_devname(cmd, dev_name(dev)))) {
 		if (_match_du_to_dev(cmd, du, dev))
 			return 1;
 	}
@@ -1534,22 +1582,6 @@ int device_ids_match_dev(struct cmd_context *cmd, struct device *dev)
  * passes the filter.
  */
 
-void device_ids_match_device_list(struct cmd_context *cmd)
-{
-	struct dev_use *du;
-
-	dm_list_iterate_items(du, &cmd->use_devices) {
-		if (du->dev)
-			continue;
-		if (!(du->dev = dev_cache_get(cmd, du->devname, NULL))) {
-			log_warn("Device not found for %s.", du->devname);
-		} else {
-			/* Should we set dev->id?  Which idtype?  Use --deviceidtype? */
-			du->dev->flags |= DEV_MATCHED_USE_ID;
-		}
-	}
-}
-
 void device_ids_match(struct cmd_context *cmd)
 {
 	struct dev_iter *iter;
@@ -1557,7 +1589,16 @@ void device_ids_match(struct cmd_context *cmd)
 	struct device *dev;
 
 	if (cmd->enable_devices_list) {
-		device_ids_match_device_list(cmd);
+		dm_list_iterate_items(du, &cmd->use_devices) {
+			if (du->dev)
+				continue;
+			if (!(du->dev = dev_cache_get_existing(cmd, du->devname, NULL))) {
+				log_warn("Device not found for %s.", du->devname);
+			} else {
+				/* Should we set dev->id?  Which idtype?  Use --deviceidtype? */
+				du->dev->flags |= DEV_MATCHED_USE_ID;
+			}
+		}
 		return;
 	}
 
@@ -1589,7 +1630,7 @@ void device_ids_match(struct cmd_context *cmd)
 		 * the du/dev pairs in preparation for using the filters.
 		 */
 		if (du->devname &&
-		    (dev = dev_cache_get(cmd, du->devname, NULL))) {
+		    (dev = dev_cache_get_existing(cmd, du->devname, NULL))) {
 			/* On successful match, du, dev, and id are linked. */
 			if (_match_du_to_dev(cmd, du, dev))
 				continue;
@@ -1710,6 +1751,13 @@ void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs,
 		if (scanned_devs && !dev_in_device_list(dev, scanned_devs))
 			continue;
 
+		/*
+		 * The matched device could not be read so we do not have
+		 * the PVID from disk and cannot verify the devices file entry.
+		 */
+		if (dev->flags & DEV_SCAN_NOT_READ)
+			continue;
+
 		/*
 		 * du and dev may have been matched, but the dev could still
 		 * have been excluded by other filters during label scan.
@@ -1717,7 +1765,7 @@ void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs,
 		 * probably wants to do something about it.
 		 */
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, dev, "persistent")) {
-			log_warn("Devices file %s is excluded by filter: %s.",
+			log_warn("Devices file %s is excluded: %s.",
 				 dev_name(dev), dev_filtered_reason(dev));
 			continue;
 		}
@@ -1792,8 +1840,18 @@ void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs,
 		if (scanned_devs && !dev_in_device_list(dev, scanned_devs))
 			continue;
 
+		/*
+		 * The matched device could not be read so we do not have
+		 * the PVID from disk and cannot verify the devices file entry.
+		 */
+		if (dev->flags & DEV_SCAN_NOT_READ)
+			continue;
+
+		if (dm_list_empty(&dev->aliases))
+			continue;
+
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, dev, "persistent")) {
-			log_warn("Devices file %s is excluded by filter: %s.",
+			log_warn("Devices file %s is excluded: %s.",
 				 dev_name(dev), dev_filtered_reason(dev));
 			/* FIXME: what if this dev is wrongly matched and should be checked below? */
 			continue;
@@ -1907,6 +1965,14 @@ void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs,
 			*device_ids_invalid = 1;
 	}
 
+	/*
+	 * When a new devname/pvid mismatch is discovered, a new search for the
+	 * pvid should be permitted (searched_devnames may exist to suppress
+	 * searching for other pvids.)
+	 */
+	if (update_file)
+		unlink_searched_devnames(cmd);
+
 	/* FIXME: for wrong devname cases, wait to write new until device_ids_find_renamed_devs? */
 
 	/*
@@ -1975,12 +2041,19 @@ void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_l
 	search_auto = !strcmp(cmd->search_for_devnames, "auto");
 
 	dm_list_iterate_items(du, &cmd->use_devices) {
-		if (du->dev)
-			continue;
 		if (!du->pvid)
 			continue;
 		if (du->idtype != DEV_ID_TYPE_DEVNAME)
 			continue;
+
+		/*
+		 * if the old incorrect devname is now a device that's
+		 * filtered and not scanned, e.g. an mpath component,
+		 * then we want to look for the pvid on a new device.
+		 */
+		if (du->dev && !du->dev->filtered_flags)
+			continue;
+
 		if (!(dil = dm_pool_zalloc(cmd->mem, sizeof(*dil))))
 			continue;
 
@@ -2005,6 +2078,11 @@ void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_l
 	 * the searched file, so a subsequent lvm command will do the search
 	 * again.  In future perhaps we could add a policy to automatically
 	 * remove a devices file entry that's not been found for some time.
+	 *
+	 * TODO: like the hint file, add a hash of all devnames to the searched
+	 * file so it can be ignored and removed if the devs/hash change.
+	 * If hints are enabled, the hints invalidation could also remove the
+	 * searched file.
 	 */
 	if (_searched_devnames_exists(cmd)) {
 		log_debug("Search for PVIDs skipped for %s", _searched_file);
@@ -2149,14 +2227,14 @@ void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_l
 	dm_list_iterate_items(dil, &search_pvids) {
 		char *dup_devname1, *dup_devname2, *dup_devname3;
 
-		if (!dil->dev) {
+		if (!dil->dev || dm_list_empty(&dil->dev->aliases)) {
 			not_found++;
 			continue;
 		}
-		found++;
 
 		dev = dil->dev;
 		devname = dev_name(dev);
+		found++;
 
 		if (!(du = get_du_for_pvid(cmd, dil->pvid))) {
 			/* shouldn't happen */
@@ -2180,7 +2258,8 @@ void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_l
 			continue;
 		}
 
-		log_warn("Devices file PVID %s updating IDNAME to %s.", dev->pvid, devname);
+		if (!noupdate)
+			log_warn("Devices file PVID %s updating IDNAME to %s.", dev->pvid, devname);
 
 		free(du->idname);
 		free(du->devname);
@@ -2208,7 +2287,7 @@ void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_l
 
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, dev, NULL)) {
 			/* I don't think this would happen */
-			log_warn("WARNING: new device %s for PVID %s does not pass filter %s.",
+			log_warn("WARNING: new device %s for PVID %s is excluded: %s.",
 				 dev_name(dev), dil->pvid, dev_filtered_reason(dev));
 			if (du) /* Should not happen 'du' is NULL */
 				du->dev = NULL;

lib/device/device_id.h

@@ -32,7 +32,6 @@ int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid,
 void device_id_pvremove(struct cmd_context *cmd, struct device *dev);
 void device_ids_match(struct cmd_context *cmd);
 int device_ids_match_dev(struct cmd_context *cmd, struct device *dev);
-void device_ids_match_device_list(struct cmd_context *cmd);
 void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs, int *device_ids_invalid, int noupdate);
 int device_ids_version_unchanged(struct cmd_context *cmd);
 void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_list, int *search_count, int noupdate);
@@ -41,6 +40,7 @@ void device_id_update_vg_uuid(struct cmd_context *cmd, struct volume_group *vg,
 
 struct dev_use *get_du_for_dev(struct cmd_context *cmd, struct device *dev);
 struct dev_use *get_du_for_pvid(struct cmd_context *cmd, const char *pvid);
+struct dev_use *get_du_for_devname(struct cmd_context *cmd, const char *devname);
 
 char *devices_file_version(void);
 int devices_file_exists(struct cmd_context *cmd);
@@ -55,5 +55,9 @@ void devices_file_exit(struct cmd_context *cmd);
 void unlink_searched_devnames(struct cmd_context *cmd);
 
 int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suffix, char *sysbuf, int sysbufsize);
+int read_sys_block_binary(struct cmd_context *cmd, struct device *dev,
+			  const char *suffix, char *sysbuf, int sysbufsize, int *retlen);
+
+int dev_has_mpath_uuid(struct cmd_context *cmd, struct device *dev, const char **idname_out);
 
 #endif

lib/device/parse_vpd.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/device/device.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <limits.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <assert.h>
+
+/*
+ * Replace series of spaces with a single _.
+ */
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int in_space = 0;
+	int retlen = 0;
+	int j = 0;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		if (!in[i])
+			break;
+		if (j >= (out_bytes - 2))
+			break;
+		/* skip leading spaces */
+		if (!retlen && (in[i] == ' '))
+			continue;
+		/* replace one or more spaces with _ */
+		if (in[i] == ' ') {
+			in_space = 1;
+			continue;
+		}
+		/* spaces are finished so insert _ */
+		if (in_space) {
+			out[j++] = '_';
+			in_space = 0;
+			retlen++;
+		}
+		out[j++] = in[i];
+		retlen++;
+	}
+	return retlen;
+}
+
+static int _to_hex(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int off = 0;
+	int num;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		num = sprintf((char *)out + off, "%02x", in[i]);
+		if (num < 0)
+			break;
+		off += num;
+		if (off + 2 >= out_bytes)
+			break;
+	}
+	return off;
+}
+
+#define ID_BUFSIZE 1024
+
+/*
+ * based on linux kernel function
+ */
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids)
+{
+	char id[ID_BUFSIZE];
+	unsigned char tmp_str[ID_BUFSIZE];
+	const unsigned char *d, *cur_id_str;
+	size_t id_len = ID_BUFSIZE;
+	int id_size = -1;
+	uint8_t cur_id_size = 0;
+
+	memset(id, 0, ID_BUFSIZE);
+	for (d = vpd_data + 4;
+	     d < vpd_data + vpd_datalen;
+	     d += d[3] + 4) {
+		memset(tmp_str, 0, sizeof(tmp_str));
+
+		switch (d[1] & 0xf) {
+		case 0x1:
+			/* T10 Vendor ID */
+			cur_id_size = d[3];
+			if (cur_id_size + 4 > id_len)
+				cur_id_size = id_len - 4;
+			cur_id_str = d + 4;
+			format_t10_id(cur_id_str, cur_id_size, tmp_str, sizeof(tmp_str));
+			id_size = snprintf(id, ID_BUFSIZE, "t10.%s", tmp_str);
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			add_wwid(id, 1, ids);
+			break;
+		case 0x2:
+			/* EUI-64 */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 12:
+				_to_hex(cur_id_str, 12, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			add_wwid(id, 2, ids);
+			break;
+		case 0x3:
+			/* NAA */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			add_wwid(id, 3, ids);
+			break;
+		case 0x8:
+			/* SCSI name string */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			if (cur_id_size >= id_len)
+				cur_id_size = id_len - 1;
+			memcpy(id, cur_id_str, cur_id_size);
+			id_size = cur_id_size;
+
+			/*
+			 * Not in the kernel version, copying multipath code,
+			 * which checks if this string begins with naa or eui
+			 * and if so does tolower() on the chars.
+			 */
+			if (!strncmp(id, "naa.", 4) || !strncmp(id, "eui.", 4)) {
+				int i;
+				for (i = 0; i < id_size; i++)
+					id[i] = tolower(id[i]);
+			}
+			add_wwid(id, 8, ids);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return id_size;
+}

lib/filters/filter-sysfs.c

@@ -15,268 +15,41 @@
 #include "lib/misc/lib.h"
 #include "lib/filters/filter.h"
 
+static int _sys_dev_block_found;
+
 #ifdef __linux__
 
-#include <sys/sysmacros.h>
-#include <dirent.h>
-
-static int _locate_sysfs_blocks(const char *sysfs_dir, char *path, size_t len,
-				unsigned *sysfs_depth)
-{
-	struct stat info;
-	unsigned i;
-	static const struct dir_class {
-		const char path[32];
-		int depth;
-	} classes[] = {
-		/*
-		 * unified classification directory for all kernel subsystems
-		 *
-		 * /sys/subsystem/block/devices
-		 * |-- sda -> ../../../devices/pci0000:00/0000:00:1f.2/host0/target0:0:0/0:0:0:0/block/sda
-		 * |-- sda1 -> ../../../devices/pci0000:00/0000:00:1f.2/host0/target0:0:0/0:0:0:0/block/sda/sda1
-		 *  `-- sr0 -> ../../../devices/pci0000:00/0000:00:1f.2/host1/target1:0:0/1:0:0:0/block/sr0
-		 *
-		 */
-		{ "subsystem/block/devices", 0 },
-
-		/*
-		 * block subsystem as a class
-		 *
-		 * /sys/class/block
-		 * |-- sda -> ../../devices/pci0000:00/0000:00:1f.2/host0/target0:0:0/0:0:0:0/block/sda
-		 * |-- sda1 -> ../../devices/pci0000:00/0000:00:1f.2/host0/target0:0:0/0:0:0:0/block/sda/sda1
-		 *  `-- sr0 -> ../../devices/pci0000:00/0000:00:1f.2/host1/target1:0:0/1:0:0:0/block/sr0
-		 *
-		 */
-		{ "class/block", 0 },
-
-		/*
-		 * old block subsystem layout with nested directories
-		 *
-		 * /sys/block/
-		 * |-- sda
-		 * |   |-- capability
-		 * |   |-- dev
-		 * ...
-		 * |   |-- sda1
-		 * |   |   |-- dev
-		 * ...
-		 * |
-		 * `-- sr0
-		 *     |-- capability
-		 *     |-- dev
-		 * ...
-		 *
-		 */
-
-		{ "block", 1 }
-	};
-
-	for (i = 0; i < DM_ARRAY_SIZE(classes); ++i)
-		if ((dm_snprintf(path, len, "%s%s", sysfs_dir, classes[i].path) >= 0) &&
-		    (stat(path, &info) == 0)) {
-			*sysfs_depth = classes[i].depth;
-			return 1;
-		}
-
-	return 0;
-}
-
-/*----------------------------------------------------------------
- * We need to store a set of dev_t.
- *--------------------------------------------------------------*/
-struct entry {
-	struct entry *next;
-	dev_t dev;
-};
-
-#define SET_BUCKETS 64
-struct dev_set {
-	struct dm_pool *mem;
-	const char *sys_block;
-	unsigned sysfs_depth;
-	int initialised;
-	struct entry *slots[SET_BUCKETS];
-};
-
-static struct dev_set *_dev_set_create(struct dm_pool *mem,
-				       const char *sys_block,
-				       unsigned sysfs_depth)
-{
-	struct dev_set *ds;
-
-	if (!(ds = dm_pool_zalloc(mem, sizeof(*ds))))
-		return NULL;
-
-	ds->mem = mem;
-	if (!(ds->sys_block = dm_pool_strdup(mem, sys_block)))
-		return NULL;
-
-	ds->sysfs_depth = sysfs_depth;
-	ds->initialised = 0;
-
-	return ds;
-}
-
-static unsigned _hash_dev(dev_t dev)
-{
-	return (major(dev) ^ minor(dev)) & (SET_BUCKETS - 1);
-}
-
-/*
- * Doesn't check that the set already contains dev.
- */
-static int _set_insert(struct dev_set *ds, dev_t dev)
-{
-	struct entry *e;
-	unsigned h = _hash_dev(dev);
-
-	if (!(e = dm_pool_alloc(ds->mem, sizeof(*e))))
-		return 0;
-
-	e->next = ds->slots[h];
-	e->dev = dev;
-	ds->slots[h] = e;
-
-	return 1;
-}
-
-static int _set_lookup(struct dev_set *ds, dev_t dev)
-{
-	unsigned h = _hash_dev(dev);
-	struct entry *e;
-
-	for (e = ds->slots[h]; e; e = e->next)
-		if (e->dev == dev)
-			return 1;
-
-	return 0;
-}
-
-/*----------------------------------------------------------------
- * filter methods
- *--------------------------------------------------------------*/
-static int _parse_dev(const char *file, FILE *fp, dev_t *result)
-{
-	unsigned major, minor;
-	char buffer[64];
-
-	if (!fgets(buffer, sizeof(buffer), fp)) {
-		log_error("Empty sysfs device file: %s", file);
-		return 0;
-	}
-
-	if (sscanf(buffer, "%u:%u", &major, &minor) != 2) {
-		log_error("Incorrect format for sysfs device file: %s.", file);
-		return 0;
-	}
-
-	*result = makedev(major, minor);
-	return 1;
-}
-
-static int _read_dev(const char *file, dev_t *result)
-{
-	int r;
-	FILE *fp;
-
-	if (!(fp = fopen(file, "r"))) {
-		log_sys_error("fopen", file);
-		return 0;
-	}
-
-	r = _parse_dev(file, fp, result);
-
-	if (fclose(fp))
-		log_sys_error("fclose", file);
-
-	return r;
-}
-
-/*
- * Recurse through sysfs directories, inserting any devs found.
- */
-static int _read_devs(struct dev_set *ds, const char *dir, unsigned sysfs_depth)
-{
-	struct dirent *d;
-	DIR *dr;
-	struct stat info;
-	char path[PATH_MAX];
-	char file[PATH_MAX];
-	dev_t dev = { 0 };
-	int r = 1;
-
-	if (!(dr = opendir(dir))) {
-		log_sys_error("opendir", dir);
-		return 0;
-	}
-
-	while ((d = readdir(dr))) {
-		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
-			continue;
-
-		if (dm_snprintf(path, sizeof(path), "%s/%s", dir,
-				 d->d_name) < 0) {
-			log_warn("WARNING: sysfs path name too long: %s in %s.",
-				 d->d_name, dir);
-			continue;
-		}
-
-		/* devices have a "dev" file */
-		if (dm_snprintf(file, sizeof(file), "%s/dev", path) < 0) {
-			log_warn("WARNING: sysfs path name too long: %s in %s.",
-				 d->d_name, dir);
-			continue;
-		}
-
-		if (!stat(file, &info)) {
-			/* recurse if we found a device and expect subdirs */
-			if (sysfs_depth)
-				_read_devs(ds, path, sysfs_depth - 1);
-
-			/* add the device we have found */
-			if (_read_dev(file, &dev))
-				_set_insert(ds, dev);
-		}
-	}
-
-	if (closedir(dr))
-		log_sys_debug("closedir", dir);
-
-	return r;
-}
-
-static int _init_devs(struct dev_set *ds)
-{
-	if (!_read_devs(ds, ds->sys_block, ds->sysfs_depth)) {
-		ds->initialised = -1;
-		return 0;
-	}
-
-	ds->initialised = 1;
-
-	return 1;
-}
-
-
 static int _accept_p(struct cmd_context *cmd, struct dev_filter *f, struct device *dev, const char *use_filter_name)
 {
-	struct dev_set *ds = (struct dev_set *) f->private;
+	char path[PATH_MAX];
+	const char *sysfs_dir;
+	struct stat info;
+
+	if (!_sys_dev_block_found)
+		return 1;
 
 	dev->filtered_flags &= ~DEV_FILTERED_SYSFS;
 
-	if (!ds->initialised)
-		_init_devs(ds);
-
-	/* Pass through if initialisation failed */
-	if (ds->initialised != 1)
+	/*
+	 * Any kind of device id other than devname has been set
+	 * using sysfs so we know that sysfs info exists for dev.
+	 */
+	if (dev->id && dev->id->idtype && (dev->id->idtype != DEV_ID_TYPE_DEVNAME))
 		return 1;
 
-	if (!_set_lookup(ds, dev->dev)) {
-		log_debug_devs("%s: Skipping (sysfs)", dev_name(dev));
-		dev->filtered_flags |= DEV_FILTERED_SYSFS;
-		return 0;
+	sysfs_dir = dm_sysfs_dir();
+	if (sysfs_dir && *sysfs_dir) {
+		if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d",
+				sysfs_dir, (int)MAJOR(dev->dev), (int)MINOR(dev->dev)) < 0) {
+			log_debug("failed to create sysfs path");
+			return 1;
+		}
+
+		if (lstat(path, &info)) {
+			log_debug_devs("%s: Skipping (sysfs)", dev_name(dev));
+			dev->filtered_flags |= DEV_FILTERED_SYSFS;
+			return 0;
+		}
 	}
 
 	return 1;
@@ -284,21 +57,34 @@ static int _accept_p(struct cmd_context *cmd, struct dev_filter *f, struct devic
 
 static void _destroy(struct dev_filter *f)
 {
-	struct dev_set *ds = (struct dev_set *) f->private;
-
 	if (f->use_count)
 		log_error(INTERNAL_ERROR "Destroying sysfs filter while in use %u times.", f->use_count);
+	free(f);
+}
 
-	dm_pool_destroy(ds->mem);
+static void _check_sys_dev_block(void)
+{
+	char path[PATH_MAX];
+	const char *sysfs_dir;
+	struct stat info;
+
+	sysfs_dir = dm_sysfs_dir();
+	if (sysfs_dir && *sysfs_dir) {
+		if (dm_snprintf(path, sizeof(path), "%sdev/block", sysfs_dir) < 0)
+			return;
+
+		if (lstat(path, &info)) {
+			log_debug("filter-sysfs disabled: /sys/dev/block not found");
+			_sys_dev_block_found = 0;
+		} else {
+			_sys_dev_block_found = 1;
+		}
+	}
 }
 
 struct dev_filter *sysfs_filter_create(void)
 {
 	const char *sysfs_dir = dm_sysfs_dir();
-	char sys_block[PATH_MAX];
-	unsigned sysfs_depth;
-	struct dm_pool *mem;
-	struct dev_set *ds;
 	struct dev_filter *f;
 
 	if (!*sysfs_dir) {
@@ -306,26 +92,15 @@ struct dev_filter *sysfs_filter_create(void)
 		return NULL;
 	}
 
-	if (!_locate_sysfs_blocks(sysfs_dir, sys_block, sizeof(sys_block), &sysfs_depth))
-		return NULL;
+	/* support old kernels that don't have this */
+	_check_sys_dev_block();
 
-	if (!(mem = dm_pool_create("sysfs", 256))) {
-		log_error("sysfs pool creation failed");
-		return NULL;
-	}
-
-	if (!(ds = _dev_set_create(mem, sys_block, sysfs_depth))) {
-		log_error("sysfs dev_set creation failed");
-		goto bad;
-	}
-
-	if (!(f = dm_pool_zalloc(mem, sizeof(*f))))
+	if (!(f = zalloc(sizeof(*f))))
 		goto_bad;
 
 	f->passes_filter = _accept_p;
 	f->destroy = _destroy;
 	f->use_count = 0;
-	f->private = ds;
 	f->name = "sysfs";
 
 	log_debug_devs("Sysfs filter initialised.");
@@ -333,7 +108,6 @@ struct dev_filter *sysfs_filter_create(void)
 	return f;
 
  bad:
-	dm_pool_destroy(mem);
 	return NULL;
 }
 

lib/format_text/archive.c

@@ -219,7 +219,7 @@ static void _remove_expired(const char *dir, const char *vgname,
 
 	sum /= 1024 * 1024;
 	if (sum > 128 || archives_size > 8192)
-		log_print_unless_silent("Consider prunning %s VG archive with more then %u MiB in %u files (check archiving is needed in lvm.conf).",
+		log_print_unless_silent("Consider pruning %s VG archive with more then %u MiB in %u files (check archiving is needed in lvm.conf).",
 					vgname, (unsigned)sum, archives_size);
 }
 

lib/label/hints.c

@@ -234,6 +234,7 @@ static int _touch_newhints(void)
 		return_0;
 	if (fclose(fp))
 		stack;
+	log_debug("newhints created");
 	return 1;
 }
 
@@ -365,7 +366,6 @@ static void _unlock_hints(struct cmd_context *cmd)
 
 void hints_exit(struct cmd_context *cmd)
 {
-	free_hints(&cmd->hints);
 	if (_hints_fd == -1)
 		return;
 	_unlock_hints(cmd);
@@ -498,6 +498,8 @@ int validate_hints(struct cmd_context *cmd, struct dm_list *hints)
 	if (!(iter = dev_iter_create(NULL, 0)))
 		return 0;
 	while ((dev = dev_iter_get(cmd, iter))) {
+		if (dm_list_empty(&dev->aliases))
+			continue;
 		if (!(hint = _find_hint_name(hints, dev_name(dev))))
 			continue;
 
@@ -505,6 +507,19 @@ int validate_hints(struct cmd_context *cmd, struct dm_list *hints)
 		if (!hint->chosen)
 			continue;
 
+		/* 
+		 * label_scan was unable to read the dev so we don't know its pvid.
+		 * Since we are unable to verify the hint is correct, it's possible
+		 * that the PVID is actually found on a different device, so don't
+		 * depend on hints. (This would also fail the following pvid check.)
+		 */
+		if (dev->flags & DEV_SCAN_NOT_READ) {
+			log_debug("Uncertain hint for unread device %d:%d %s",
+				  major(hint->devt), minor(hint->devt), dev_name(dev));
+			ret = 0;
+			continue;
+		}
+
 		if (strcmp(dev->pvid, hint->pvid)) {
 			log_debug("Invalid hint device %d:%d %s pvid %s had hint pvid %s",
 				  major(hint->devt), minor(hint->devt), dev_name(dev),
@@ -1391,6 +1406,16 @@ int get_hints(struct cmd_context *cmd, struct dm_list *hints_out, int *newhints,
 		log_debug("get_hints: needs refresh");
 		free_hints(&hints_list);
 
+		/*
+		 * This is not related to hints, and is probably unnecessary,
+		 * but it could possibly help.  When hints become invalid it's
+		 * usually becaues devs on the system have changed, and that
+		 * also means that a missing devices file entry might be found
+		 * by searching devices again.  (the searched_devnames
+		 * mechanism should eventually be replaced)
+		 */
+		unlink_searched_devnames(cmd);
+
 		if (!_lock_hints(cmd, LOCK_EX, NONBLOCK))
 			return 0;
 

lib/label/label.c

@@ -458,7 +458,6 @@ static int _scan_dev_open(struct device *dev)
 	const char *name;
 	const char *modestr;
 	struct stat sbuf;
-	int retried = 0;
 	int flags = 0;
 	int fd, di;
 
@@ -478,14 +477,23 @@ static int _scan_dev_open(struct device *dev)
 		return 0;
 	}
 
+ next_name:
 	/*
 	 * All the names for this device (major:minor) are kept on
 	 * dev->aliases, the first one is the primary/preferred name.
+	 *
+	 * The default name preferences in dev-cache mean that the first
+	 * name in dev->aliases is not a symlink for scsi devices, but is
+	 * the /dev/mapper/ symlink for mpath devices.
+	 *
+	 * If preferred names are set to symlinks, should this
+	 * first attempt to open using a non-symlink?
+	 *
+	 * dm_list_first() returns NULL if the list is empty.
 	 */
 	if (!(name_list = dm_list_first(&dev->aliases))) {
-		/* Shouldn't happen */
-		log_error("Device open %s %d:%d has no path names.",
-			  dev_name(dev), (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		log_error("Device open %d:%d has no path names.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
 		return 0;
 	}
 	name_sl = dm_list_item(name_list, struct dm_str_list);
@@ -513,50 +521,34 @@ static int _scan_dev_open(struct device *dev)
 		modestr = "ro";
 	}
 
-retry_open:
-
 	fd = open(name, flags, 0777);
-
 	if (fd < 0) {
 		if ((errno == EBUSY) && (flags & O_EXCL)) {
 			log_error("Can't open %s exclusively.  Mounted filesystem?",
 				  dev_name(dev));
+			return 0;
 		} else {
-			int major, minor;
-
 			/*
-			 * Shouldn't happen, if it does, print stat info to help figure
-			 * out what's wrong.
+			 * drop name from dev->aliases and use verify_aliases to
+			 * drop any other invalid aliases before retrying open with
+			 * any remaining valid paths.
 			 */
-
-			major = (int)MAJOR(dev->dev);
-			minor = (int)MINOR(dev->dev);
-
-			log_error("Device open %s %d:%d failed errno %d", name, major, minor, errno);
-
-			if (stat(name, &sbuf)) {
-				log_debug_devs("Device open %s %d:%d stat failed errno %d",
-					       name, major, minor, errno);
-			} else if (sbuf.st_rdev != dev->dev) {
-				log_debug_devs("Device open %s %d:%d stat %d:%d does not match.",
-					       name, major, minor,
-					       (int)MAJOR(sbuf.st_rdev), (int)MINOR(sbuf.st_rdev));
-			}
-
-			if (!retried) {
-				/*
-				 * FIXME: remove this, the theory for this retry is that
-				 * there may be a udev race that we can sometimes mask by
-				 * retrying.  This is here until we can figure out if it's
-				 * needed and if so fix the real problem.
-				 */
-				usleep(5000);
-				log_debug_devs("Device open %s retry", dev_name(dev));
-				retried = 1;
-				goto retry_open;
-			}
+			log_debug("Drop alias for %d:%d failed open %s (%d)",
+				  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), name, errno);
+			dev_cache_failed_path(dev, name);
+			dev_cache_verify_aliases(dev);
+			goto next_name;
 		}
-		return 0;
+	}
+
+	/* Verify that major:minor from the path still match dev. */
+	if ((fstat(fd, &sbuf) < 0) || (sbuf.st_rdev != dev->dev)) {
+		log_warn("Invalid path %s for device %d:%d, trying different path.",
+			 name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		(void)close(fd);
+		dev_cache_failed_path(dev, name);
+		dev_cache_verify_aliases(dev);
+		goto next_name;
 	}
 
 	dev->flags |= DEV_IN_BCACHE;
@@ -604,37 +596,6 @@ static int _scan_dev_close(struct device *dev)
 	return 1;
 }
 
-static void _drop_bad_aliases(struct device *dev)
-{
-	struct dm_str_list *strl, *strl2;
-	const char *name;
-	struct stat sbuf;
-	int major = (int)MAJOR(dev->dev);
-	int minor = (int)MINOR(dev->dev);
-	int bad;
-
-	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
-		name = strl->str;
-		bad = 0;
-
-		if (stat(name, &sbuf)) {
-			bad = 1;
-			log_debug_devs("Device path check %d:%d %s stat failed errno %d",
-					major, minor, name, errno);
-		} else if (sbuf.st_rdev != dev->dev) {
-			bad = 1;
-			log_debug_devs("Device path check %d:%d %s stat %d:%d does not match.",
-				       major, minor, name,
-				       (int)MAJOR(sbuf.st_rdev), (int)MINOR(sbuf.st_rdev));
-		}
-
-		if (bad) {
-			log_debug_devs("Device path check %d:%d dropping path %s.", major, minor, name);
-			dev_cache_failed_path(dev, name);
-		}
-	}
-}
-
 // Like bcache_invalidate, only it throws any dirty data away if the
 // write fails.
 static void _invalidate_di(struct bcache *cache, int di)
@@ -662,10 +623,8 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 	char headers_buf[HEADERS_BUF_SIZE];
 	struct dm_list wait_devs;
 	struct dm_list done_devs;
-	struct dm_list reopen_devs;
 	struct device_list *devl, *devl2;
 	struct block *bb;
-	int retried_open = 0;
 	int scan_read_errors = 0;
 	int scan_process_errors = 0;
 	int scan_failed_count = 0;
@@ -676,7 +635,6 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 
 	dm_list_init(&wait_devs);
 	dm_list_init(&done_devs);
-	dm_list_init(&reopen_devs);
 
 	log_debug_devs("Scanning %d devices for VG info", dm_list_size(devs));
 
@@ -686,6 +644,8 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 
 	dm_list_iterate_items_safe(devl, devl2, devs) {
 
+		devl->dev->flags &= ~DEV_SCAN_NOT_READ;
+
 		/*
 		 * If we prefetch more devs than blocks in the cache, then the
 		 * cache will wait for earlier reads to complete, toss the
@@ -698,9 +658,10 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 
 		if (!_in_bcache(devl->dev)) {
 			if (!_scan_dev_open(devl->dev)) {
-				log_debug_devs("Scan failed to open %s.", dev_name(devl->dev));
+				log_debug_devs("Scan failed to open %d:%d %s.",
+					       (int)MAJOR(devl->dev->dev), (int)MINOR(devl->dev->dev), dev_name(devl->dev));
 				dm_list_del(&devl->list);
-				dm_list_add(&reopen_devs, &devl->list);
+				devl->dev->flags |= DEV_SCAN_NOT_READ;
 				continue;
 			}
 		}
@@ -724,6 +685,7 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 			log_debug_devs("Scan failed to read %s.", dev_name(devl->dev));
 			scan_read_errors++;
 			scan_failed_count++;
+			devl->dev->flags |= DEV_SCAN_NOT_READ;
 			lvmcache_del_dev(devl->dev);
 			if (bb)
 				bcache_put(bb);
@@ -782,41 +744,6 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 	if (!dm_list_empty(devs))
 		goto scan_more;
 
-	/*
-	 * We're done scanning all the devs.  If we failed to open any of them
-	 * the first time through, refresh device paths and retry.  We failed
-	 * to open the devs on the reopen_devs list.
-	 *
-	 * FIXME: it's not clear if or why this helps.
-	 */
-	if (!dm_list_empty(&reopen_devs)) {
-		if (retried_open) {
-			/* Don't try again. */
-			scan_failed_count += dm_list_size(&reopen_devs);
-			dm_list_splice(&done_devs, &reopen_devs);
-			goto out;
-		}
-		retried_open = 1;
-
-		dm_list_iterate_items_safe(devl, devl2, &reopen_devs) {
-			_drop_bad_aliases(devl->dev);
-
-			if (dm_list_empty(&devl->dev->aliases)) {
-				log_warn("WARNING: Scan ignoring device %d:%d with no paths.",
-					 (int)MAJOR(devl->dev->dev),
-					 (int)MINOR(devl->dev->dev));
-					 
-				dm_list_del(&devl->list);
-				lvmcache_del_dev(devl->dev);
-				scan_failed_count++;
-			}
-		}
-
-		/* Put devs that failed to open back on the original list to retry. */
-		dm_list_splice(devs, &reopen_devs);
-		goto scan_more;
-	}
-out:
 	log_debug_devs("Scanned devices: read errors %d process errors %d failed %d",
 			scan_read_errors, scan_process_errors, scan_failed_count);
 
@@ -873,7 +800,7 @@ static int _setup_bcache(void)
 	}
 
 	if (!(scan_bcache = bcache_create(BCACHE_BLOCK_SIZE_IN_SECTORS, cache_blocks, ioe))) {
-		log_error("Failed to create bcache with %d cache blocks.", cache_blocks);
+		log_error("Failed to set up io layer with %d blocks.", cache_blocks);
 		return 0;
 	}
 
@@ -891,7 +818,7 @@ static int _setup_bcache(void)
 
 #define BASE_FD_COUNT 32 /* Number of open files we want apart from devs */
 
-static void _prepare_open_file_limit(struct cmd_context *cmd, unsigned int num_devs)
+void prepare_open_file_limit(struct cmd_context *cmd, unsigned int num_devs)
 {
 #ifdef HAVE_PRLIMIT
 	struct rlimit old = { 0 }, new;
@@ -1088,7 +1015,7 @@ int label_scan(struct cmd_context *cmd)
 	 * data to invalidate.)
 	 */
 	if (!(iter = dev_iter_create(NULL, 0))) {
-		log_error("Scanning failed to get devices.");
+		log_error("Failed to get device list.");
 		return 0;
 	}
 	while ((dev = dev_iter_get(cmd, iter))) {
@@ -1113,6 +1040,10 @@ int label_scan(struct cmd_context *cmd)
 	 * filter", and this result needs to be cleared (wiped) so that the
 	 * complete set of filters (including those that require data) can be
 	 * checked in _process_block, where headers have been read.
+	 *
+	 * FIXME: devs that are filtered with data in _process_block
+	 * are not moved to the filtered_devs list like devs filtered
+	 * here without data.  Does that have any effect?
 	 */
 	log_debug_devs("Filtering devices to scan (nodata)");
 
@@ -1165,7 +1096,7 @@ int label_scan(struct cmd_context *cmd)
 	 * which we want to keep open) is higher than the current
 	 * soft limit.
 	 */
-	_prepare_open_file_limit(cmd, dm_list_size(&scan_devs));
+	prepare_open_file_limit(cmd, dm_list_size(&scan_devs));
 
 	/*
 	 * Do the main scan.
@@ -1207,8 +1138,6 @@ int label_scan(struct cmd_context *cmd)
 			 (unsigned long long)want_size_kb);
 	}
 
-	dm_list_init(&cmd->hints);
-
 	/*
 	 * If we're using hints to limit which devs we scanned, verify
 	 * that those hints were valid, and if not we need to scan the
@@ -1220,18 +1149,16 @@ int label_scan(struct cmd_context *cmd)
 			_scan_list(cmd, cmd->filter, &all_devs, 0, NULL);
 			/* scan_devs are the devs that have been scanned */
 			dm_list_splice(&scan_devs, &all_devs);
-			free_hints(&hints_list);
 			using_hints = 0;
 			create_hints = 0;
 			/* invalid hints means a new dev probably appeared and
 			   we should search for any missing pvids again. */
 			unlink_searched_devnames(cmd);
-		} else {
-			/* The hints may be used by another device iteration. */
-			dm_list_splice(&cmd->hints, &hints_list);
 		}
 	}
 
+	free_hints(&hints_list);
+
 	/*
 	 * Check if the devices_file content is up to date and
 	 * if not update it.
@@ -1439,7 +1366,7 @@ void label_scan_invalidate_lv(struct cmd_context *cmd, struct logical_volume *lv
 	if (lv_info(cmd, lv, 0, &lvinfo, 0, 0) && lvinfo.exists) {
 		/* FIXME: Still unclear what is it supposed to find */
 		devt = MKDEV(lvinfo.major, lvinfo.minor);
-		if ((dev = dev_cache_get_by_devt(cmd, devt, NULL, NULL)))
+		if ((dev = dev_cache_get_by_devt(cmd, devt)))
 			label_scan_invalidate(dev);
 	}
 }
@@ -1561,10 +1488,24 @@ int label_scan_open_rw(struct device *dev)
 
 int label_scan_reopen_rw(struct device *dev)
 {
+	const char *name;
 	int flags = 0;
 	int prev_fd = dev->bcache_fd;
 	int fd;
 
+	if (dm_list_empty(&dev->aliases)) {
+		log_error("Cannot reopen rw device %d:%d with no valid paths di %d fd %d.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev->bcache_di, dev->bcache_fd);
+		return 0;
+	}
+
+	name = dev_name(dev);
+	if (!name || name[0] != '/') {
+		log_error("Cannot reopen rw device %d:%d with no valid name di %d fd %d.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev->bcache_di, dev->bcache_fd);
+		return 0;
+	}
+
 	if (!(dev->flags & DEV_IN_BCACHE)) {
 		if ((dev->bcache_fd != -1) || (dev->bcache_di != -1)) {
 			/* shouldn't happen */
@@ -1594,7 +1535,7 @@ int label_scan_reopen_rw(struct device *dev)
 	flags |= O_NOATIME;
 	flags |= O_RDWR;
 
-	fd = open(dev_name(dev), flags, 0777);
+	fd = open(name, flags, 0777);
 	if (fd < 0) {
 		log_error("Failed to open rw %s errno %d di %d fd %d.",
 			  dev_name(dev), errno, dev->bcache_di, dev->bcache_fd);

lib/label/label.h

@@ -134,4 +134,6 @@ void dev_invalidate(struct device *dev);
 void dev_set_last_byte(struct device *dev, uint64_t offset);
 void dev_unset_last_byte(struct device *dev);
 
+void prepare_open_file_limit(struct cmd_context *cmd, unsigned int num_devs);
+
 #endif

lib/locking/lvmlockd.c

@@ -272,6 +272,8 @@ static void _lockd_retrive_vg_pv_list(struct volume_group *vg,
 
 	i = 0;
 	dm_list_iterate_items(pvl, &vg->pvs) {
+		if (!pvl->pv->dev || dm_list_empty(&pvl->pv->dev->aliases))
+			continue;
 		lock_pvs->path[i] = strdup(pv_dev_name(pvl->pv));
 		if (!lock_pvs->path[i]) {
 			log_error("Fail to allocate PV path for VG %s", vg->name);
@@ -341,6 +343,8 @@ static void _lockd_retrive_lv_pv_list(struct volume_group *vg,
 
 	dm_list_iterate_items(pvl, &vg->pvs) {
 		if (lv_is_on_pv(lv, pvl->pv)) {
+			if (!pvl->pv->dev || dm_list_empty(&pvl->pv->dev->aliases))
+				continue;
 			lock_pvs->path[i] = strdup(pv_dev_name(pvl->pv));
 			if (!lock_pvs->path[i]) {
 				log_error("Fail to allocate PV path for LV %s/%s",

lib/log/log.c

@@ -892,6 +892,7 @@ uint32_t log_journal_str_to_val(const char *str)
 		return LOG_JOURNAL_OUTPUT;
 	if (!strcasecmp(str, "debug"))
 		return LOG_JOURNAL_DEBUG;
+	log_warn("Ignoring unrecognized journal value.");
 	return 0;
 }
 

lib/metadata/mirror.c

@@ -1231,14 +1231,23 @@ int remove_mirrors_from_segments(struct logical_volume *lv,
 const char *get_pvmove_pvname_from_lv_mirr(const struct logical_volume *lv_mirr)
 {
 	struct lv_segment *seg;
+	struct device *dev;
 
 	dm_list_iterate_items(seg, &lv_mirr->segments) {
 		if (!seg_is_mirrored(seg))
 			continue;
-		if (seg_type(seg, 0) == AREA_PV)
-			return dev_name(seg_dev(seg, 0));
-		if (seg_type(seg, 0) == AREA_LV)
-			return dev_name(seg_dev(first_seg(seg_lv(seg, 0)), 0));
+		if (seg_type(seg, 0) == AREA_PV) {
+			dev = seg_dev(seg, 0);
+			if (!dev || dm_list_empty(&dev->aliases))
+				return NULL;
+			return dev_name(dev);
+		}
+		if (seg_type(seg, 0) == AREA_LV) {
+			dev = seg_dev(first_seg(seg_lv(seg, 0)), 0);
+			if (!dev || dm_list_empty(&dev->aliases))
+				return NULL;
+			return dev_name(dev);
+		}
 	}
 
 	return NULL;

lib/metadata/pv_list.c

@@ -152,6 +152,11 @@ static int _create_pv_entry(struct dm_pool *mem, struct pv_list *pvl,
 	struct pv_list *new_pvl = NULL, *pvl2;
 	struct dm_list *pe_ranges;
 
+	if (!pvl->pv->dev || dm_list_empty(&pvl->pv->dev->aliases)) {
+		log_error("Failed to create PV entry for missing device.");
+		return 0;
+	}
+
 	pvname = pv_dev_name(pvl->pv);
 	if (allocatable_only && !(pvl->pv->status & ALLOCATABLE_PV)) {
 		log_warn("WARNING: Physical volume %s not allocatable.", pvname);

lib/metadata/vdo_manip.c

@@ -392,7 +392,9 @@ struct logical_volume *convert_vdo_pool_lv(struct logical_volume *data_lv,
 
 	/* Format data LV as VDO volume */
 	if (format) {
-		if (!_format_vdo_pool_data_lv(data_lv, vtp, &vdo_logical_size)) {
+		if (test_mode()) {
+			log_verbose("Test mode: Skipping formating of VDO pool volume.");
+		} else if (!_format_vdo_pool_data_lv(data_lv, vtp, &vdo_logical_size)) {
 			log_error("Cannot format VDO pool volume %s.", display_lvname(data_lv));
 			return NULL;
 		}

lib/metadata/vg.c

@@ -679,6 +679,11 @@ int vgreduce_single(struct cmd_context *cmd, struct volume_group *vg,
 		return r;
 	}
 
+	if (!pv->dev || dm_list_empty(&pv->dev->aliases)) {
+		log_error("No device found for PV.");
+		return r;
+	}
+
 	log_debug("vgreduce_single VG %s PV %s", vg->name, pv_dev_name(pv));
 
 	if (pv_pe_alloc_count(pv)) {

lib/report/columns.h

@@ -108,6 +108,7 @@ FIELD(LVS, lv, TIM, "RTime", lvid, 26, lvtimeremoved, lv_time_removed, "Removal
 FIELD(LVS, lv, STR, "Host", lvid, 10, lvhost, lv_host, "Creation host of the LV, if known.", 0)
 FIELD(LVS, lv, STR_LIST, "Modules", lvid, 0, modules, lv_modules, "Kernel device-mapper modules required for this LV.", 0)
 FIELD(LVS, lv, BIN, "Historical", lvid, 0, lvhistorical, lv_historical, "Set if the LV is historical.", 0)
+FIELD(LVS, lv, NUM, "WCacheBlkSize", lvid, 0, writecache_block_size, writecache_block_size, "The writecache block size", 0)
 /*
  * End of LVS type fields
  */

lib/report/properties.c

@@ -353,6 +353,8 @@ GET_PV_STR_PROPERTY_FN(pv_device_id_type, pv->device_id_type)
 #define _writecache_writeback_blocks_get prop_not_implemented_get
 #define _writecache_error_set prop_not_implemented_set
 #define _writecache_error_get prop_not_implemented_get
+#define _writecache_block_size_set prop_not_implemented_set
+#define _writecache_block_size_get prop_not_implemented_get
 
 #define _vdo_operating_mode_set prop_not_implemented_set
 #define _vdo_operating_mode_get prop_not_implemented_get

lib/report/report.c

@@ -3346,6 +3346,26 @@ static int _integritymismatches_disp(struct dm_report *rh __attribute__((unused)
 	return _field_set_value(field, "", &GET_TYPE_RESERVED_VALUE(num_undef_64));
 }
 
+static int _writecache_block_size_disp(struct dm_report *rh __attribute__((unused)),
+				   struct dm_pool *mem,
+				   struct dm_report_field *field,
+				   const void *data,
+				   void *private __attribute__((unused)))
+{
+	struct logical_volume *lv = (struct logical_volume *) data;
+	uint32_t bs = 0;
+
+	if (lv_is_writecache(lv)) {
+		struct lv_segment *seg = first_seg(lv);
+		bs = seg->writecache_block_size;
+	}
+
+	if (!bs)
+		return dm_report_field_int32(rh, field, &GET_TYPE_RESERVED_VALUE(num_undef_32));
+
+	return dm_report_field_uint32(rh, field, &bs);
+}
+
 static int _datapercent_disp(struct dm_report *rh, struct dm_pool *mem,
 			     struct dm_report_field *field,
 			     const void *data, void *private)

man/Makefile.in

@@ -33,7 +33,7 @@ CMIRRORDMAN = cmirrord.8
 LVMDBUSDMAN = lvmdbusd.8
 
 MAN5=lvm.conf.5
-MAN7=lvmsystemid.7 lvmreport.7 lvmraid.7
+MAN7=lvmsystemid.7 lvmreport.7 lvmraid.7 lvmautoactivation.7
 
 MAN8=lvm.8 lvmdump.8 lvm-fullreport.8 lvm-lvpoll.8 \
      lvcreate.8 lvchange.8 lvmconfig.8 lvconvert.8 lvdisplay.8 \

man/lvdisplay.8_pregen

@@ -61,8 +61,6 @@ and more, using a more compact and configurable output format.
 .br
 [    \fB--readonly\fP ]
 .br
-[    \fB--reportformat\fP \fBbasic\fP|\fBjson\fP ]
-.br
 [    \fB--segments\fP ]
 .br
 [    \fB--separator\fP \fIString\fP ]
@@ -332,16 +330,6 @@ device-mapper kernel driver, so this option is unable to report whether
 or not LVs are actually in use.
 .
 .HP
-\fB--reportformat\fP \fBbasic\fP|\fBjson\fP
-.br
-Overrides current output format for reports which is defined globally by
-the report/output_format setting in \fBlvm.conf\fP(5).
-\fBbasic\fP is the original format with columns and rows.
-If there is more than one report per command, each report is prefixed
-with the report name for identification. \fBjson\fP produces report
-output in JSON format. See \fBlvmreport\fP(7) for more information.
-.
-.HP
 \fB--segments\fP
 .br
 .

man/lvm.8_main

@@ -579,6 +579,7 @@ Prepends source file name and code line number with libdm debugging.
 .BR lvmraid (7),
 .BR lvmthin (7),
 .BR lvmcache (7),
+.BR lvmautoactivation (7),
 .P
 .BR dmsetup (8),
 .BR dmstats (8),

man/lvmautoactivation.7_main

@@ -0,0 +1,314 @@
+.TH "LVMAUTOACTIVATION" "7" "LVM TOOLS #VERSION#" "Red Hat, Inc" "\""
+.
+.SH NAME
+.
+lvmautoactivation \(em LVM autoactivation
+.
+.SH DESCRIPTION
+.
+Autoactivation is the activation of LVs performed automatically by the
+system in response to LVM devices being attached to the machine.  When all
+PVs in a VG have been attached, the VG is complete, and LVs in the VG are
+activated.
+.P
+Autoactivation of VGs, or specific LVs, can be prevented using vgchange or
+lvchange --setautoactivation n.  The lvm.conf auto_activation_volume_list
+is another way to limit autoactivation.
+.
+.SS event autoactivation
+.P
+The most common form of autoactivation is "event based", in which complete
+VGs are activated in response to uevents which occur during system startup
+or at any time after the system has started.  Another form of
+autoactivation is "static" in which complete VGs are activated at a fixed
+point during system startup by a systemd service, and not in response to
+events.  This can be controlled with the lvm.conf setting
+event_activation.
+.P
+Event based autoactivation is driven by udev, udev rules, and systemd.
+When a device is attached to a machine, a uevent is generated by the
+kernel to notify userspace of the new device.  systemd-udev runs udev
+rules to process the new device.  Udev rules use blkid to identify the
+device as an LVM PV and then execute the lvm-specific udev rule for the
+device, which triggers autoactivation.
+.P
+There are two variations of event baed autoactivation that may be used on
+a system, depending on the LVM udev rule that is installed (found in
+/lib/udev/rules.d/.)  The following summarizes the steps in each rule
+which lead to autoactivation:
+.P
+.B 69-dm-lvm-metad.rules
+.
+.IP \[bu] 2
+device /dev/name with major:minor X:Y is attached to the machine
+.
+.IP \[bu] 2
+systemd/udev runs blkid to identify /dev/name as an LVM PV
+.
+.IP \[bu] 2
+udev rule 69-dm-lvm-metad.rules is run for /dev/name
+.
+.IP \[bu] 2
+the lvm udev rule runs the systemd service lvm2-pvscan@X:Yservice
+.
+.IP \[bu] 2
+the lvm2-pvscan service runs:
+.br
+pvscan --cache -aay --major X --minor Y
+.
+.IP \[bu] 2
+pvscan reads the device, records that the PV is online
+(see pvs_online), and checks if the VG is complete.
+.
+.IP \[bu] 2
+if the VG is complete, pvscan creates the vgs_online temp file,
+and activates the VG.
+.
+.IP \[bu] 2
+the activation command output can be seen from
+systemctl status lvm2-pvscan*
+.P
+.B 69-dm-lvm.rules
+.
+.IP \[bu] 2
+device /dev/name with major:minor X:Y is attached to the machine
+.
+.IP \[bu] 2
+systemd/udev runs blkid to identify /dev/name as an LVM PV
+.
+.IP \[bu] 2
+udev rule 69-dm-lvm.rules is run for /dev/name
+.
+.IP \[bu] 2
+the lvm udev rule runs:
+.br
+pvscan --cache --listvg --checkcomplete --vgonline
+.br
+--autoactivation event --udevoutput --journal=output /dev/name
+.
+.IP \[bu] 2
+pvscan reads the device, records that the PV is online
+(see pvs_online), and checks if the VG is complete.
+.
+.IP \[bu] 2
+if the VG is complete, pvscan creates the vgs_online temp file,
+and prints the name of the VG for the udev rule to import:
+LVM_VG_NAME_COMPLETE='vgname'
+.
+.IP \[bu] 2
+if the lvm udev rule sees LVM_VG_NAME_COMPLETE from pvscan,
+it activates the VG using a transient systemd service named
+lvm-activate-<vgname>.
+.
+.IP \[bu] 2
+the lvm-activate-<vgname> service runs
+.br
+vgchange -aay --autoactivation event <vgname>
+.
+.IP \[bu] 2
+the activation command output can be seen from
+journalctl -u lvm-activate-<vgname>
+.P
+.
+.SS pvscan options
+.P
+.B --cache
+.br
+Read the <device> arg (and only that device), and record that
+the PV is online by creating the /run/lvm/pvs_online/<pvid>
+file containing the name of the VG and the device for the PV.
+.P
+.B -aay
+.br
+Activate the VG from the pvscan command
+(includes implicit --checkcomplete and --vgonline.)
+.P
+.B --checkcomplete
+.br
+Check if the VG is complete, i.e. all PVs are present on
+the system, by checking /run/lvm/pvs_online/<pvid> files.
+.P
+.B --vgonline
+.br
+Create /run/lvm/vgs_online/<vgname> if the VG is complete
+(used to ensure only one command performs activation.)
+.P
+.B --autoactivation event
+.br
+Inform the command it is used for event based autoactivation.
+.P
+.B --listvg
+.br
+Print the name of the VG using the device.
+.P
+.B --udevoutput
+.br
+Only print output that can be imported to the udev rule,
+using the udev environment key format, i.e. NAME='value'.
+.P
+.B --journal=output
+.br
+Send standard command output to the journal (when stdout
+is reserved for udev output.)
+.P
+.SS run files
+.P
+Autoactivation commands use a number of temp files in /run/lvm (with the
+expectation that /run is cleared between boots.)
+.P
+.B pvs_online
+.br
+pvscan --cache creates a file here for each PV that is attached.  The file
+is named with the PVID and contains the VG name and device information.
+The existence of the file is used to determine when all PVs for a given VG
+are present.  The device information in these files is also used to
+optimize locating devices for a VG when the VG is activated.
+.P
+.B pvs_lookup
+.br
+pvscan --cache creates a file here named for a VG (if one doesn't already
+exist.)  The file contains a list of PVIDs in the VG.  This is needed when
+a PV is processed which has no VG metadata, in which case the list of
+PVIDs from the lookup file is used to check if the VG is complete.
+.P
+.B vgs_online
+.br
+The first activation command (pvscan or vgchange) to create a file here,
+named for the VG, will activate the VG.  This resolves a race when
+concurrent commands attempt to activate a VG at once.
+.
+.SS static autoactivation
+.P
+When event autoactivation is disabled by setting lvm.conf
+event_activation=0, autoactivation is performed at one or more static
+points during system startup. At these points, a vgchange -aay command is
+run to activate complete VGs from devices that are present on the system
+at that time. pvscan commands (and lvm2-pvscan services) do not perform
+autoactivation in this mode.  pvscan commands may still be run from
+uevents but will do nothing when they read the event_activation=0 setting.
+.P
+The static vgchange -aay commands are run by three systemd services at
+three points during startup: lvm2-activation-early, lvm2-activation, and
+lvm2-activation-net. These static activation services are "generated
+services", so the service files are created at run time by the
+lvm2-activation-generator command (run by systemd).
+lvm2-activation-generator creates the services if lvm.conf
+event_activation=0.
+.P
+The limitation of this method is that devices may not be attached to the
+system (or set up) at a reliable point in time during startup, and they
+may not be present when the services run vgchange.  In this case, the VGs
+will not be autoactivated.  So, the timing of device attachment/setup
+determines whether static autoactivation will produce the same results as
+event autoactivation.  For this reason, static autoactivation is not
+recommended.
+.P
+Sometimes, static autoactivation is mistakenly expected to disable all
+autoactivation of particular VGs. This may appear to be effective if those
+VGs are slow to be attached or set up. But, the only correct and reliable
+way to disable autoactivation is using vgchange/lvchange
+--setautoactivation n, or lvm.conf auto_activation_volume_list.
+.
+.SH EXAMPLES
+.P
+VG "vg" contains two PVs:
+.nf
+$ pvs -o name,vgname,uuid /dev/sdb /dev/sdc
+  PV         VG PV UUID                               
+  /dev/sdb   vg 1uKpaT-lFOZ-NLHX-j4jI-OBi1-QpdE-HZ5hZY
+  /dev/sdc   vg 5J3tM8-aIPe-2vbd-DBe7-bvRq-TGj0-DaKV2G
+.fi
+.P
+use of --cache:
+.nf
+$ pvscan --cache /dev/sdb
+  pvscan[12922] PV /dev/sdb online.
+$ pvscan --cache /dev/sdc
+  pvscan[12923] PV /dev/sdc online.
+
+$ cat /run/lvm/pvs_online/1uKpaTlFOZNLHXj4jIOBi1QpdEHZ5hZY 
+8:16
+vg:vg
+dev:/dev/sdb
+$ cat /run/lvm/pvs_online/5J3tM8aIPe2vbdDBe7bvRqTGj0DaKV2G 
+8:32
+vg:vg
+dev:/dev/sdc
+.fi
+.P
+use of -aay:
+.nf
+$ pvscan --cache -aay /dev/sdb
+  pvscan[12935] PV /dev/sdb online, VG vg incomplete (need 1).
+$ pvscan --cache -aay /dev/sdc
+  pvscan[12936] PV /dev/sdc online, VG vg is complete.
+  pvscan[12936] VG vg run autoactivation.
+  1 logical volume(s) in volume group "vg" now active
+
+$ cat /run/lvm/pvs_online/1uKpaTlFOZNLHXj4jIOBi1QpdEHZ5hZY 
+8:16
+vg:vg
+dev:/dev/sdb
+$ cat /run/lvm/pvs_online/5J3tM8aIPe2vbdDBe7bvRqTGj0DaKV2G 
+8:32
+vg:vg
+dev:/dev/sdc
+$ ls /run/lvm/vgs_online/vg 
+/run/lvm/vgs_online/vg
+.fi
+.P
+use of --listvg:
+.nf
+$ pvscan --cache --listvg /dev/sdb
+  VG vg
+$ pvscan --cache --listvg /dev/sdc
+  VG vg
+
+$ cat /run/lvm/pvs_online/1uKpaTlFOZNLHXj4jIOBi1QpdEHZ5hZY 
+8:16
+vg:vg
+dev:/dev/sdb
+$ cat /run/lvm/pvs_online/5J3tM8aIPe2vbdDBe7bvRqTGj0DaKV2G 
+8:32
+vg:vg
+dev:/dev/sdc
+.fi
+.P
+use of --checkcomplete:
+.nf
+$ pvscan --cache --listvg --checkcomplete --vgonline /dev/sdb
+  pvscan[12996] PV /dev/sdb online, VG vg incomplete (need 1).
+  VG vg incomplete
+$ pvscan --cache --listvg --checkcomplete --vgonline /dev/sdc
+  pvscan[12997] PV /dev/sdc online, VG vg is complete.
+  VG vg complete
+.fi
+.P
+use of --udevoutput:
+.nf
+$ pvscan --cache --listvg --checkcomplete --vgonline --udevoutput /dev/sdb
+LVM_VG_NAME_INCOMPLETE='vg'
+$ pvscan --cache --listvg --checkcomplete --vgonline --udevoutput /dev/sdc
+LVM_VG_NAME_COMPLETE='vg'
+.fi
+.P
+use of --listlvs:
+.nf
+$ lvs -o name,devices vg
+  LV    Devices                
+  lvol0 /dev/sdb(0)            
+  lvol1 /dev/sdc(0)            
+  lvol2 /dev/sdb(1),/dev/sdc(1)
+
+$ pvscan --cache --listlvs --checkcomplete /dev/sdb
+  pvscan[13288] PV /dev/sdb online, VG vg incomplete (need 1).
+  VG vg incomplete
+  LV vg/lvol0 complete
+  LV vg/lvol2 incomplete
+$ pvscan --cache --listlvs --checkcomplete /dev/sdc
+  pvscan[13289] PV /dev/sdc online, VG vg is complete.
+  VG vg complete
+  LV vg/lvol1 complete
+  LV vg/lvol2 complete
+.fi
+

man/lvmcache.7_main

@@ -240,6 +240,22 @@ The writecache block size should be chosen to match the xfs sectsz value.
 It is also possible to specify a sector size of 4096 to mkfs.xfs when
 creating the file system.  In this case the writecache block size of 4096
 can be used.
+.P
+The writecache block size is displayed by the command:
+.br
+lvs -o writecacheblocksize VG/LV
+.P
+.SS dm-writecache memory usage
+.P
+The amount of main system memory used by dm-writecache can be a factor
+when selecting the writecache cachevol size and the writecache block size.
+.P
+.IP \[bu] 2
+writecache block size 4096: each 100 GiB of writecache cachevol uses
+slighly over 2 GiB of system memory.
+.IP \[bu] 2
+writecache block size 512: each 100 GiB of writecache cachevol uses
+a little over 16 GiB of system memory.
 .
 .SS dm-writecache settings
 .

man/lvmvdo.7_main

@@ -132,6 +132,19 @@ that can keep 100% incompressible data there.
 # lvconvert --type vdo-pool -n vdo0 -V10G vg/ExistingLV
 .fi
 .
+.SS \n+[step]. Change the compression and deduplication of a VDOPoolLV
+.
+Disable or enable the compression and deduplication for VDOPoolLV
+(the volume that maintains all VDO LV(s) associated with it).
+.P
+.B lvchange --compression y|n --deduplication y|n VG/VDOPoolLV
+.P
+.I Example
+.nf
+# lvchange --compression n  vg/vdopool0
+# lvchange --deduplication y vg/vdopool1
+.fi
+.
 .SS \n+[step]. Change the default settings used for creating a VDOPoolLV
 .
 VDO allows to set a large variety of options. Lots of these settings
@@ -173,17 +186,27 @@ EOF
 # lvcreate --vdo -L10G --config 'allocation/vdo_cpu_threads=4' vg/vdopool1
 .fi
 .
-.SS \n+[step]. Change the compression and deduplication of a VDOPoolLV
+.SS \n+[step]. Set or change VDO settings with option --vdosettings
 .
-Disable or enable the compression and deduplication for VDOPoolLV
-(the volume that maintains all VDO LV(s) associated with it).
-.P
-.B lvchange --compression y|n --deduplication y|n VG/VDOPoolLV
+Use the form 'option=value' or 'option1=value option2=value',
+or repeat --vdosettings for each option being set.
+Options are listed in the Example section above, for the full description see
+.BR lvm.conf (5).
+Options can omit 'vdo_' and 'vdo_use_' prefixes and all its underscores.
+So i.e.  vdo_use_metadata_hints=1  and  metadatahints=1 are equivalent.
+To change the option for an already existing VDOPoolLV use
+.BR lvchange (8)
+command. However not all option can be changed.
+Only compression and deduplication options can be also changed for an active VDO LV.
+Lowest priority options are specified with configuration file,
+then with --vdosettings and highest are expliction option --compression
+and --deduplication.
 .P
 .I Example
+.P
 .nf
-# lvchange --compression n  vg/vdopool0
-# lvchange --deduplication y vg/vdopool1
+# lvcreate --vdo -L10G --vdosettings 'ack_threads=1 hash_zone_threads=2' vg/vdopool0
+# lvchange --vdosettings 'bio_threads=2 deduplication=1' vg/vdopool0
 .fi
 .
 .SS \n+[step]. Checking the usage of VDOPoolLV

man/pvdisplay.8_pregen

@@ -61,8 +61,6 @@ and more, using a more compact and configurable output format.
 .br
 [    \fB--readonly\fP ]
 .br
-[    \fB--reportformat\fP \fBbasic\fP|\fBjson\fP ]
-.br
 [    \fB--separator\fP \fIString\fP ]
 .br
 [    \fB--shared\fP ]
@@ -320,16 +318,6 @@ device-mapper kernel driver, so this option is unable to report whether
 or not LVs are actually in use.
 .
 .HP
-\fB--reportformat\fP \fBbasic\fP|\fBjson\fP
-.br
-Overrides current output format for reports which is defined globally by
-the report/output_format setting in \fBlvm.conf\fP(5).
-\fBbasic\fP is the original format with columns and rows.
-If there is more than one report per command, each report is prefixed
-with the report name for identification. \fBjson\fP produces report
-output in JSON format. See \fBlvmreport\fP(7) for more information.
-.
-.HP
 \fB-S\fP|\fB--select\fP \fIString\fP
 .br
 Select objects for processing and reporting based on specified criteria.

man/pvscan.8_des

@@ -4,56 +4,47 @@ like
 or
 .BR pvdisplay (8).
 .P
-When the --cache and -aay options are used, pvscan records which PVs are
-available on the system, and activates LVs in completed VGs.  A VG is
-complete when pvscan sees that the final PV in the VG has appeared.  This
-is used by event-based system startup (systemd, udev) to activate LVs.
-.P
-The four main variations of this are:
+When --cache is used, pvscan updates runtime lvm state on the system, or
+with -aay performs autoactivation.
 .P
 .B pvscan --cache
 .I device
 .P
-If device is present, lvm adds a record that the PV on device is online.
+If device is present, lvm records that the PV on device is online.
 If device is not present, lvm removes the online record for the PV.
-In most cases, the pvscan will only read the named devices.
-.P
-.B pvscan --cache -aay
-.IR device ...
-.P
-This begins by performing the same steps as above.  Afterward, if the VG
-for the specified PV is complete, then pvscan will activate LVs in the VG
-(the same as vgchange -aay vgname would do.)
+pvscan only reads the named device.
 .P
 .B pvscan --cache
 .P
-This first clears all existing PV online records, then scans all devices
-on the system, adding PV online records for any PVs that are found.
+Updates the runtime state for all lvm devices.
+.P
+.B pvscan --cache -aay
+.I device
+.P
+Performs the --cache steps for the device, then checks if the VG using the
+device is complete.  If so, LVs in the VG are autoactivated, the same as
+vgchange -aay vgname would do.  (A device name may be replaced with major
+and minor numbers.)
 .P
 .B pvscan --cache -aay
 .P
-This begins by performing the same steps as pvscan --cache.  Afterward, it
-activates LVs in any complete VGs.
+Performs the --cache steps for all devices, then autoactivates any complete VGs.
 .P
-To prevent devices from being scanned by pvscan --cache, add them
-to
-.BR lvm.conf (5)
-.B devices/global_filter.
-For more information, see:
-.br
-.B lvmconfig --withcomments devices/global_filter
+.B pvscan --cache --listvg|--listlvs
+.I device
 .P
-Auto-activation of VGs or LVs can be enabled/disabled using:
-.br
+Performs the --cache steps for the device, then prints the name of the VG
+using the device, or the names of LVs using the device.  --checkcomplete
+is usually included to check if all PVs for the VG or LVs are online.
+When this command is called by a udev rule, the output must conform to
+udev rule specifications (see --udevoutput.)  The udev rule will use the
+results to perform autoactivation.
+.P
+Autoactivation of VGs or LVs can be enabled/disabled using vgchange or
+lvchange with --setautoactivation y|n, or by adding names to
 .BR lvm.conf (5)
 .B activation/auto_activation_volume_list
 .P
-For more information, see:
-.br
-.B lvmconfig --withcomments activation/auto_activation_volume_list
-.P
-To disable auto-activation, explicitly set this list to an empty list,
-i.e. auto_activation_volume_list = [ ].
-.P
-When this setting is undefined (e.g. commented), then all LVs are
-auto-activated.
+See
+.BR lvmautoactivation (7)
+for more information about how pvscan is used for autoactivation.

man/pvscan.8_pregen

@@ -91,59 +91,50 @@ like
 or
 .BR pvdisplay (8).
 .P
-When the --cache and -aay options are used, pvscan records which PVs are
-available on the system, and activates LVs in completed VGs.  A VG is
-complete when pvscan sees that the final PV in the VG has appeared.  This
-is used by event-based system startup (systemd, udev) to activate LVs.
-.P
-The four main variations of this are:
+When --cache is used, pvscan updates runtime lvm state on the system, or
+with -aay performs autoactivation.
 .P
 .B pvscan --cache
 .I device
 .P
-If device is present, lvm adds a record that the PV on device is online.
+If device is present, lvm records that the PV on device is online.
 If device is not present, lvm removes the online record for the PV.
-In most cases, the pvscan will only read the named devices.
-.P
-.B pvscan --cache -aay
-.IR device ...
-.P
-This begins by performing the same steps as above.  Afterward, if the VG
-for the specified PV is complete, then pvscan will activate LVs in the VG
-(the same as vgchange -aay vgname would do.)
+pvscan only reads the named device.
 .P
 .B pvscan --cache
 .P
-This first clears all existing PV online records, then scans all devices
-on the system, adding PV online records for any PVs that are found.
+Updates the runtime state for all lvm devices.
+.P
+.B pvscan --cache -aay
+.I device
+.P
+Performs the --cache steps for the device, then checks if the VG using the
+device is complete.  If so, LVs in the VG are autoactivated, the same as
+vgchange -aay vgname would do.  (A device name may be replaced with major
+and minor numbers.)
 .P
 .B pvscan --cache -aay
 .P
-This begins by performing the same steps as pvscan --cache.  Afterward, it
-activates LVs in any complete VGs.
+Performs the --cache steps for all devices, then autoactivates any complete VGs.
 .P
-To prevent devices from being scanned by pvscan --cache, add them
-to
-.BR lvm.conf (5)
-.B devices/global_filter.
-For more information, see:
-.br
-.B lvmconfig --withcomments devices/global_filter
+.B pvscan --cache --listvg|--listlvs
+.I device
 .P
-Auto-activation of VGs or LVs can be enabled/disabled using:
-.br
+Performs the --cache steps for the device, then prints the name of the VG
+using the device, or the names of LVs using the device.  --checkcomplete
+is usually included to check if all PVs for the VG or LVs are online.
+When this command is called by a udev rule, the output must conform to
+udev rule specifications (see --udevoutput.)  The udev rule will use the
+results to perform autoactivation.
+.P
+Autoactivation of VGs or LVs can be enabled/disabled using vgchange or
+lvchange with --setautoactivation y|n, or by adding names to
 .BR lvm.conf (5)
 .B activation/auto_activation_volume_list
 .P
-For more information, see:
-.br
-.B lvmconfig --withcomments activation/auto_activation_volume_list
-.P
-To disable auto-activation, explicitly set this list to an empty list,
-i.e. auto_activation_volume_list = [ ].
-.P
-When this setting is undefined (e.g. commented), then all LVs are
-auto-activated.
+See
+.BR lvmautoactivation (7)
+for more information about how pvscan is used for autoactivation.
 .
 .SH USAGE
 .

man/vgdisplay.8_pregen

@@ -58,8 +58,6 @@ and more, using a more compact and configurable output format.
 .br
 [    \fB--readonly\fP ]
 .br
-[    \fB--reportformat\fP \fBbasic\fP|\fBjson\fP ]
-.br
 [    \fB--shared\fP ]
 .br
 [    \fB--separator\fP \fIString\fP ]
@@ -312,16 +310,6 @@ device-mapper kernel driver, so this option is unable to report whether
 or not LVs are actually in use.
 .
 .HP
-\fB--reportformat\fP \fBbasic\fP|\fBjson\fP
-.br
-Overrides current output format for reports which is defined globally by
-the report/output_format setting in \fBlvm.conf\fP(5).
-\fBbasic\fP is the original format with columns and rows.
-If there is more than one report per command, each report is prefixed
-with the report name for identification. \fBjson\fP produces report
-output in JSON format. See \fBlvmreport\fP(7) for more information.
-.
-.HP
 \fB-S\fP|\fB--select\fP \fIString\fP
 .br
 Select objects for processing and reporting based on specified criteria.

scripts/Makefile.in

@@ -92,6 +92,7 @@ install_systemd_generators:
 install_systemd_units:	install_dbus_service
 	@echo "    [INSTALL] systemd_units"
 	$(Q) $(INSTALL_DIR) $(systemd_unit_dir)
+	$(Q) $(INSTALL_DATA) lvm2-pvscan.service $(systemd_unit_dir)/lvm2-pvscan@.service
 ifeq ("@BUILD_DMEVENTD@", "yes")
 	$(Q) $(INSTALL_DATA) dm_event_systemd_red_hat.socket $(systemd_unit_dir)/dm-event.socket
 	$(Q) $(INSTALL_DATA) dm_event_systemd_red_hat.service $(systemd_unit_dir)/dm-event.service

scripts/lvm_import_vdo.sh

@@ -125,15 +125,6 @@ get_kb_size_with_unit_() {
 	esac
 }
 
-get_mb_size_with_unit_() {
-	case "$1" in
-	*[mM]) echo $(( ${1%[mM]} )) ;;
-	*[gG]) echo $(( ${1%[gG]} * 1024 )) ;;
-	*[tT]) echo $(( ${1%[tT]} * 1024 * 1024 )) ;;
-	*[pP]) echo $(( ${1%[pP]} * 1024 * 1024 * 1024 )) ;;
-	esac
-}
-
 # Figure out largest possible extent size usable for VG
 # $1   physical size
 # $2   logical size
@@ -328,12 +319,12 @@ allocation {
 	vdo_use_deduplication = $(get_enabled_value_ "$vdo_deduplication")
 	vdo_use_metadata_hints=1
 	vdo_minimum_io_size = $vdo_logicalBlockSize
-	vdo_block_map_cache_size_mb = $(get_mb_size_with_unit_ "$vdo_blockMapCacheSize")
+	vdo_block_map_cache_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_blockMapCacheSize") / 1024 ))
 	vdo_block_map_period = $vdo_blockMapPeriod
 	vdo_check_point_frequency = $vdo_indexCfreq
 	vdo_use_sparse_index = $(get_enabled_value_ "$vdo_indexSparse")
 	vdo_index_memory_size_mb = $(awk "BEGIN {print $vdo_indexMemory * 1024}")
-	vdo_slab_size_mb = $(get_mb_size_with_unit_ "$vdo_blockMapCacheSize")
+	vdo_slab_size_mb = $(( $(get_kb_size_with_unit_ "$vdo_blockMapCacheSize") / 1024 ))
 	vdo_ack_threads = $vdo_ackThreads
 	vdo_bio_threads = $vdo_bioThreads
 	vdo_bio_rotation = $vdo_bioRotationInterval

spec/packages.inc

@@ -88,6 +88,7 @@ fi
  %{_sbindir}/lvmpolld
 %endif
 %{_mandir}/man5/lvm.conf.5.gz
+%{_mandir}/man7/lvmautoactivation.7.gz
 %{_mandir}/man7/lvmsystemid.7.gz
 %{_mandir}/man7/lvmreport.7.gz
 %{_mandir}/man7/lvmraid.7.gz
@@ -163,7 +164,7 @@ fi
 %endif
 %if %{enable_udev}
  %{_udevdir}/11-dm-lvm.rules
- %{_udevdir}/69-dm-lvm.rules
+ %{_udevdir}/69-dm-lvm-metad.rules
 %endif
 %dir %{_sysconfdir}/lvm
 %ghost %{_sysconfdir}/lvm/cache/.cache
@@ -189,6 +190,7 @@ fi
  %{_tmpfilesdir}/%{name}.conf
  %{_unitdir}/blk-availability.service
  %{_unitdir}/lvm2-monitor.service
+ %{_unitdir}/lvm2-pvscan@.service
  %attr(555, -, -) %{_prefix}/lib/systemd/system-generators/lvm2-activation-generator
  %if %{have_service lvmpolld}
   %{_unitdir}/lvm2-lvmpolld.service

test/shell/devicesfile-basic.sh

@@ -104,6 +104,11 @@ not ls "$DFDIR/system.devices"
 vgs --devicesfile test.devices $vg1
 not vgs --devicesfile test.devices $vg2
 
+# misspelled override name fails
+not vgs --devicesfile doesnotexist $vg1
+not vgs --devicesfile doesnotexist $vg2
+not vgs --devicesfile doesnotexist
+
 # devicesfile and devices cannot be used together
 not vgs --devicesfile test.devices --devices "$dev1","$dev1" $vg1
 
@@ -283,7 +288,7 @@ not ls "$RUNDIR/lvm/pvs_online/$PVID3"
 # arg in devices list
 _clear_online_files
 pvscan --devices "$dev3" --cache -aay "$dev3"
-pvscan --devices "$dev4","$dev3" --cache -aay "$dev4"
+pvscan --devices "$dev4" --cache -aay "$dev4"
 check lv_field $vg2/$lv2 lv_active "active"
 vgchange -an $vg2
 

test/shell/devicesfile-devname.sh

@@ -545,4 +545,86 @@ grep "$PVID2" "$DF" |tee out
 grep "$dev2" out
 not grep "$dev1" out
 
+vgchange -an $vg1
+vgchange -an $vg2
+vgremove -ff $vg1
+vgremove -ff $vg2
+
+# bz 2119473
+
+aux lvmconf "devices/search_for_devnames = \"none\""
+sed -e "s|DEVNAME=$dev1|DEVNAME=.|" "$ORIG" > tmp1.devices
+sed -e "s|IDNAME=$dev1|IDNAME=.|" tmp1.devices > "$DF"
+pvs
+lvmdevices
+pvcreate -ff --yes --uuid "$PVID1" --norestorefile $dev1
+grep "$PVID1" "$DF" |tee out
+grep "DEVNAME=$dev1" out
+grep "IDNAME=$dev1" out
+aux lvmconf "devices/search_for_devnames = \"auto\""
+
+# devnames change so the new devname now refers to a filtered device,
+# e.g. an mpath or md component, which is not scanned
+
+wait_md_create() {
+        local md=$1
+
+        while :; do
+                if ! grep "$(basename $md)" /proc/mdstat; then
+                        echo "$md not ready"
+                        cat /proc/mdstat
+                        sleep 2
+                else
+                        break
+                fi
+        done
+        echo "$md" > WAIT_MD_DEV
+}
+
+aux wipefs_a "$dev1"
+aux wipefs_a "$dev2"
+aux wipefs_a "$dev3"
+aux wipefs_a "$dev4"
+
+mddev="/dev/md33"
+not grep $mddev /proc/mdstat || skip
+
+rm "$DF"
+touch "$DF"
+vgcreate $vg1 "$dev1" "$dev2"
+cat "$DF"
+cp "$DF" "$ORIG"
+
+# PVID with dashes for matching pvs -o+uuid output
+OPVID1=`pvs "$dev1" --noheading -o uuid | awk '{print $1}'`
+OPVID2=`pvs "$dev2" --noheading -o uuid | awk '{print $1}'`
+
+# PVID without dashes for matching devices file fields
+PVID1=`pvs "$dev1" --noheading -o uuid | tr -d - | awk '{print $1}'`
+PVID2=`pvs "$dev2" --noheading -o uuid | tr -d - | awk '{print $1}'`
+
+mdadm --create --metadata=1.0 "$mddev" --level 1 --raid-devices=2 "$dev3" "$dev4"
+wait_md_create "$mddev"
+
+sed -e "s|DEVNAME=$dev1|DEVNAME=$dev3|" "$ORIG" > tmp1.devices
+sed -e "s|IDNAME=$dev1|IDNAME=$dev3|" tmp1.devices > "$DF"
+cat "$DF"
+pvs -o+uuid |tee out
+grep "$dev1" out
+grep "$dev2" out
+grep "$OPVID1" out
+grep "$OPVID2" out
+not grep "$dev3" out
+not grep "$dev4" out
+
+grep "$dev1" "$DF"
+grep "$dev2" "$DF"
+grep "$PVID1" "$DF"
+grep "$PVID2" "$DF"
+not grep "$dev3" "$DF"
+not grep "$dev4" "$DF"
+
+mdadm --stop "$mddev"
+aux udev_wait
+
 vgremove -ff $vg1

test/shell/devicesfile-realdevs.sh

@@ -423,7 +423,7 @@ sed "s/$pvid1/badpvid/" "$DF.orig" |tee $DF
 not grep $pvid1 $DF
 grep $did1 $DF
 
-lvmdevices --check 2>&1|tee out
+not lvmdevices --check 2>&1|tee out
 grep $dev1 out
 grep badpvid out
 grep $pvid1 out
@@ -493,7 +493,7 @@ rm $DF
 d1=$(basename $dev1)
 d3=$(basename $dev3)
 sed "s/$d1/$d3/" "$DF.orig" |tee $DF
-lvmdevices --check 2>&1 |tee out
+not lvmdevices --check 2>&1 |tee out
 grep $dev1 out
 
 lvmdevices --update
@@ -515,7 +515,7 @@ sed "s/$d1/tmp/" "$DF.orig" |tee ${DF}_1
 sed "s/$d2/$d1/" "${DF}_1" |tee ${DF}_2
 sed "s/tmp/$d2/" "${DF}_2" |tee $DF
 rm ${DF}_1 ${DF}_2
-lvmdevices --check 2>&1 |tee out
+not lvmdevices --check 2>&1 |tee out
 grep $dev1 out
 grep $dev2 out
 
@@ -536,7 +536,7 @@ rm $DF
 d1=$(basename $dev1)
 d3=$(basename $dev3)
 sed "s/$d1/$d3/" "$DF.orig" |tee $DF
-lvmdevices --check 2>&1 |tee out
+not lvmdevices --check 2>&1 |tee out
 grep $dev1 out
 
 pvs -o+uuid,deviceid | grep $vg |tee out

test/shell/duplicate-pvs-multipath.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Red Hat, Inc. All rights reserved.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions
+# of the GNU General Public License v.2.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+test_description='duplicate pv detection of mpath components using wwid'
+
+SKIP_WITH_LVMPOLLD=1
+SKIP_WITH_LVMLOCKD=1
+
+. lib/inittest
+
+# FIXME: skip until mpath/scsi_debug cleanup works after a failure
+skip
+
+modprobe --dry-run scsi_debug || skip
+multipath -l || skip
+multipath -l | grep scsi_debug && skip
+
+# FIXME: setting multipath_component_detection=0 now also disables
+# the wwid-based mpath component detection, so this test will need
+# to find another way to disable only the filter-mpath code (using
+# sysfs and multipath/wwids) while keeping the code enabled that
+# eliminates duplicates based on their matching wwids which this
+# tries to test.
+
+# Prevent wwids from being used for filtering.
+aux lvmconf 'devices/multipath_wwids_file = "/dev/null"'
+# Need to use /dev/mapper/mpath
+aux lvmconf 'devices/dir = "/dev"'
+aux lvmconf 'devices/scan = "/dev"'
+# Could set filter to $MP and the component /dev/sd devs
+aux lvmconf "devices/filter = [ \"a|.*|\" ]"
+aux lvmconf "devices/global_filter = [ \"a|.*|\" ]"
+
+modprobe scsi_debug dev_size_mb=100 num_tgts=1 vpd_use_hostno=0 add_host=4 delay=20 max_luns=2 no_lun_0=1
+sleep 2
+
+multipath -r
+sleep 2
+
+MPB=$(multipath -l | grep scsi_debug | cut -f1 -d ' ')
+echo $MPB
+MP=/dev/mapper/$MPB
+echo $MP
+
+pvcreate $MP
+vgcreate $vg1 $MP
+lvcreate -l1 $vg1
+vgchange -an $vg1
+
+pvs |tee out
+grep $MP out
+for i in $(grep -H scsi_debug /sys/block/sd*/device/model | cut -f4 -d /); do
+	not grep /dev/$i out;
+done
+
+vgchange -an $vg1
+vgremove -y $vg1
+
+sleep 2
+multipath -f $MP
+sleep 1
+rmmod scsi_debug

test/shell/losetup-partscan.sh

@@ -33,6 +33,8 @@ aux udev_wait
 ls -la "${LOOP}"*
 test -e "${LOOP}p1"
 
+aux lvmconf 'devices/scan = "/dev"'
+
 aux extend_filter "a|$LOOP|"
 aux extend_devices "$LOOP"
 

test/shell/lvchange-vdo.sh

@@ -48,9 +48,17 @@ check grep_dmsetup status $vg-vdopool-vpool " online online "
 lvchange --compression n --deduplication n $vg/vdopool
 check grep_dmsetup status $vg-vdopool-vpool " offline offline "
 
+# --vdosettings needs inactive LV
+not lvchange --vdosettings 'ack_threads=8' $vg/vdopool
 
 lvchange -an $vg/$lv1
 
+# With inactive vdo-pool changes are applied
+# explicit option --compression has highest priority
+lvchange --vdosettings 'ack_threads=5 compression=0' --compression y $vg/vdopool
+check lv_field $vg/$lv1 vdo_ack_threads "5"
+check lv_field $vg/$lv1 vdo_compression "enabled"
+
 # Test activation
 lvchange -aly $vg/$lv1
 check active $vg $lv1

test/shell/lvconvert-vdo.sh

@@ -28,12 +28,12 @@ lvcreate -L5G -n $lv1 $vg
 not lvconvert --type vdo-pool $vg/$lv1 |& tee out
 grep "WARNING" out
 
-
-lvconvert -y --type vdo-pool $vg/$lv1
+# Check --vdosettings is also applied to converted vdo-pool
+lvconvert -y --type vdo-pool --vdosettings 'ack_threads=5' $vg/$lv1
+check lv_field $vg/$lv1 vdo_ack_threads "5"
 lvremove -f $vg
 
-
-# 
+#
 lvcreate -L5G -n $lv1 $vg
 lvconvert -y --vdopool $vg/$lv1
 lvremove -f $vg

test/shell/lvcreate-vdo.sh

@@ -79,8 +79,12 @@ not fsck -n "$DM_DEV_DIR/mapper/$vg-${lv2}"
 
 lvremove -ff $vg
 
+# Unknown settings does not pass
+# TODO: try to catch this in parser and 'fail'
+not lvcreate --type vdo --vdosettings 'ack_Xthreads=4' -L10G -V1T -ky -n $lv1 $vg
 
-lvcreate --type vdo -L10G -V1T -ky -n $lv1 $vg
+lvcreate --type vdo --vdosettings 'ack_threads=4' -L10G -V1T -ky -n $lv1 $vg
+check lv_field $vg/$lv1 vdo_ack_threads "4"
 lvs -a $vg
 lvremove -ff $vg
 

test/shell/multipath-config.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Red Hat, Inc. All rights reserved.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions
+# of the GNU General Public License v.2.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+test_description='using multipath blacklist'
+
+SKIP_WITH_LVMPOLLD=1
+SKIP_WITH_LVMLOCKD=1
+
+. lib/inittest
+
+# FIXME: don't run this test by default because it destroys the
+# local multipath config, the timing of multipath/dm/lvm interactions
+# is fragile, and there's insufficient cleanup after a test fails.
+skip
+
+systemctl stop multipathd
+multipath -F || true
+rm /etc/multipath/wwids || true
+rmmod scsi_debug || true
+rm /etc/multipath/conf.d/lvmtest.conf || true
+
+modprobe --dry-run scsi_debug || skip
+multipath -l || skip
+multipath -l | grep scsi_debug && skip
+ls /etc/multipath/wwids && skip
+
+# Need to use /dev/mapper/mpath
+aux lvmconf 'devices/dir = "/dev"'
+aux lvmconf 'devices/scan = "/dev"'
+# Could set filter to $MP and the component /dev/sd devs
+aux lvmconf "devices/filter = [ \"a|.*|\" ]"
+aux lvmconf "devices/global_filter = [ \"a|.*|\" ]"
+
+modprobe scsi_debug dev_size_mb=16 num_tgts=1
+sleep 2
+
+# Get scsi device name created by scsi_debug.
+# SD = sdh
+# SD_DEV = /dev/sdh
+
+SD=$(grep -H scsi_debug /sys/block/sd*/device/model | cut -f4 -d /);
+echo $SD
+SD_DEV=/dev/$SD
+echo $SD_DEV
+
+# if multipath claimed SD, then io will fail
+#dd if=$SD_DEV of=/dev/null bs=4k count=1 iflag=direct
+#dd if=/dev/zero of=$SD_DEV bs=4k count=1 oflag=direct
+
+# check if multipathd claimed the scsi dev when it appears and create mp dm device
+sleep 2
+multipath -l
+# create the mp dm device
+multipath $SD_DEV
+
+# Get mpath device name created by multipath.
+# MP = mpatha
+# MP_DEV = /dev/maper/mpatha
+
+MP=$(multipath -l | grep scsi_debug | cut -f1 -d ' ')
+echo $MP
+MP_DEV=/dev/mapper/$MP
+echo $MP_DEV
+
+dd if=$MP_DEV of=/dev/null bs=4k count=1 iflag=direct
+dd if=/dev/zero of=$MP_DEV bs=4k count=1 oflag=direct
+
+# Get wwid for the mp and sd dev.
+WWID=$(multipath -l $MP_DEV | head -1 | awk '{print $2}' | tr -d ')' | tr -d '(')
+echo $WWID
+
+grep $WWID /etc/multipath/wwids
+
+pvcreate $MP_DEV
+vgcreate $vg1 $MP_DEV
+
+not pvs $SD_DEV
+pvs $MP_DEV
+
+# remove mpath dm device then check that SD_DEV is
+# filtered based on /etc/multipath/wwids instead of
+# based on sysfs holder
+multipath -f $MP
+sleep 2
+not pvs $SD_DEV
+multipath $SD_DEV
+sleep 2
+multipath -l | grep $SD
+
+#
+# Add the wwid to the blacklist, then restart multipath
+# so the sd dev should no longer be used by multipath,
+# but the sd dev wwid is still in /etc/multipath/wwids.
+#
+
+mkdir /etc/multipath/conf.d/ || true
+rm -f /etc/multipath/conf.d/lvmtest.conf
+
+cat <<EOF > "/etc/multipath/conf.d/lvmtest.conf"
+blacklist {
+	wwid $WWID
+}
+EOF
+
+cat /etc/multipath/conf.d/lvmtest.conf
+
+multipath -r
+sleep 2
+
+grep $WWID /etc/multipath/wwids
+
+multipath -l |tee out
+not grep $SD out
+not grep $MP out
+not grep $WWID out
+
+not pvs $MP_DEV
+pvs $SD_DEV
+vgs $vg1
+
+#
+# Add the wwid to the blacklist_exceptions, in addition
+# to the blacklist, then restart multipath so the
+# sd dev should again be used by multipath.
+#
+
+rm -f /etc/multipath/conf.d/lvmtest.conf
+
+cat <<EOF > "/etc/multipath/conf.d/lvmtest.conf"
+blacklist {
+wwid $WWID
+}
+blacklist_exceptions {
+wwid $WWID
+}
+EOF
+
+cat /etc/multipath/conf.d/lvmtest.conf
+
+multipath -r
+sleep 2
+
+grep $WWID /etc/multipath/wwids
+
+multipath -l |tee out
+grep $SD out
+grep $MP out
+grep $WWID out
+
+pvs $MP_DEV
+not pvs $SD_DEV
+vgs $vg1
+lvs $vg1
+
+sleep 2
+vgremove -ff $vg1
+sleep 2
+multipath -f $MP
+rm /etc/multipath/conf.d/lvmtest.conf
+rm /etc/multipath/wwids
+sleep 1
+rmmod scsi_debug

test/shell/pv-ext-flags.sh

@@ -57,9 +57,11 @@ dd if="$dev1" of=dev1_backup bs=1M
 
 # pvcreate and pvremove can be forced even if the PV is marked as used
 pvremove -ff -y "$dev1"
+lvmdevices --deldev "$dev1" || true
 dd if=dev1_backup of="$dev1" bs=1M
 pvcreate -ff -y "$dev1"
 dd if=dev1_backup of="$dev1" bs=1M
+lvmdevices --adddev "$dev1" || true
 
 # prepare a VG with $dev1 and $dev both having 1 MDA
 aux enable_dev "$dev2"
@@ -116,9 +118,11 @@ dd if="$dev1" of=dev1_backup bs=1M
 
 # pvcreate and pvremove can be forced even if the PV is marked as used
 pvremove -ff -y "$dev1"
+lvmdevices --deldev "$dev1" || true
 dd if=dev1_backup of="$dev1" bs=1M
 pvcreate -ff -y "$dev1"
 dd if=dev1_backup of="$dev1" bs=1M
+lvmdevices --adddev "$dev1" || true
 
 # prepare a VG with $dev1 and $dev both having 1 MDA
 aux enable_dev "$dev2"

test/shell/udev-pvscan-vgchange.sh

@@ -1,403 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Red Hat, Inc. All rights reserved.
-#
-# This copyrighted material is made available to anyone wishing to use,
-# modify, copy, or redistribute it subject to the terms and conditions
-# of the GNU General Public License v.2.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-test_description='udev rule and systemd unit run vgchange'
-
-SKIP_WITH_LVMPOLLD=1
-SKIP_WITH_LVMLOCKD=1
-
-. lib/inittest
-
-#
-# $ cat /tmp/devs
-# /dev/sdb
-# /dev/sdc
-# /dev/sdd
-#
-# Specify this file as LVM_TEST_DEVICE_LIST=/tmp/devs
-# when running the test.
-#
-# This test will wipe these devices.
-#
-
-if [ -z ${LVM_TEST_DEVICE_LIST+x} ]; then echo "LVM_TEST_DEVICE_LIST is unset" && skip; else echo "LVM_TEST_DEVICE_LIST is set to '$LVM_TEST_DEVICE_LIST'"; fi
-
-test -e "$LVM_TEST_DEVICE_LIST" || skip
-
-num_devs=$(cat $LVM_TEST_DEVICE_LIST | wc -l)
-
-RUNDIR="/run"
-test -d "$RUNDIR" || RUNDIR="/var/run"
-PVS_ONLINE_DIR="$RUNDIR/lvm/pvs_online"
-VGS_ONLINE_DIR="$RUNDIR/lvm/vgs_online"
-PVS_LOOKUP_DIR="$RUNDIR/lvm/pvs_lookup"
-
-_clear_online_files() {
-	# wait till udev is finished
-	aux udev_wait
-	rm -f "$PVS_ONLINE_DIR"/*
-	rm -f "$VGS_ONLINE_DIR"/*
-	rm -f "$PVS_LOOKUP_DIR"/*
-}
-
-test -d "$PVS_ONLINE_DIR" || mkdir -p "$PVS_ONLINE_DIR"
-test -d "$VGS_ONLINE_DIR" || mkdir -p "$VGS_ONLINE_DIR"
-test -d "$PVS_LOOKUP_DIR" || mkdir -p "$PVS_LOOKUP_DIR"
-_clear_online_files
-
-aux prepare_real_devs
-
-aux lvmconf 'devices/dir = "/dev"'
-aux lvmconf 'devices/use_devicesfile = 1'
-DFDIR="$LVM_SYSTEM_DIR/devices"
-DF="$DFDIR/system.devices"
-mkdir $DFDIR || true
-not ls $DF
-
-get_real_devs
-
-wipe_all() {
-	for dev in "${REAL_DEVICES[@]}"; do
-		wipefs -a $dev
-	done
-}
-
-# udevadm trigger runs udev rule which runs systemd-run --no-wait vgchange -aay
-# Because of --no-wait, we need to wait for the transient systemd
-# service to be gone before checking the effects of the vgchange.
-
-wait_lvm_activate() {
-	local vgw=$1
-	local wait=0
-
-	while systemctl status lvm-activate-$vgw > /dev/null && test "$wait" -le 30; do
-		sleep .2
-		wait=$(( wait + 1 ))
-	done
-}
-
-# Test requires 3 devs
-test $num_devs -gt 2 || skip
-BDEV1=$(basename "$dev1")
-BDEV2=$(basename "$dev2")
-BDEV3=$(basename "$dev3")
-
-wipe_all
-touch $DF
-for dev in "${REAL_DEVICES[@]}"; do
-	pvcreate $dev
-done
-
-# 1 dev, 1 vg, 1 lv
-
-vgcreate $vg1 "$dev1"
-lvcreate -l1 -an -n $lv1 $vg1 "$dev1"
-
-PVID1=$(pvs "$dev1" --noheading -o uuid | tr -d - | awk '{print $1}')
-
-_clear_online_files
-udevadm trigger --settle -c add /sys/block/$BDEV1
-
-wait_lvm_activate $vg1
-
-ls "$RUNDIR/lvm/pvs_online/$PVID1"
-ls "$RUNDIR/lvm/vgs_online/$vg1"
-journalctl -u lvm-activate-$vg1 | tee out || true
-grep "now active" out
-check lv_field $vg1/$lv1 lv_active "active"
-
-vgchange -an $vg1
-vgremove -y $vg1
-
-
-# 2 devs, 1 vg, 2 lvs
-
-vgcreate $vg2 "$dev1" "$dev2"
-lvcreate -l1 -an -n $lv1 $vg2 "$dev1"
-lvcreate -l1 -an -n $lv2 $vg2 "$dev2"
-
-PVID1=$(pvs "$dev1" --noheading -o uuid | tr -d - | awk '{print $1}')
-PVID2=$(pvs "$dev2" --noheading -o uuid | tr -d - | awk '{print $1}')
-
-_clear_online_files
-
-udevadm trigger --settle -c add /sys/block/$BDEV1
-ls "$RUNDIR/lvm/pvs_online/$PVID1"
-not ls "$RUNDIR/lvm/vgs_online/$vg2"
-journalctl -u lvm-activate-$vg2 | tee out || true
-not grep "now active" out
-check lv_field $vg2/$lv1 lv_active ""
-check lv_field $vg2/$lv2 lv_active ""
-
-udevadm trigger --settle -c add /sys/block/$BDEV2
-ls "$RUNDIR/lvm/pvs_online/$PVID2"
-ls "$RUNDIR/lvm/vgs_online/$vg2"
-
-wait_lvm_activate $vg2
-
-journalctl -u lvm-activate-$vg2 | tee out || true
-grep "now active" out
-check lv_field $vg2/$lv1 lv_active "active"
-check lv_field $vg2/$lv2 lv_active "active"
-
-vgchange -an $vg2
-vgremove -y $vg2
-
-
-# 3 devs, 1 vg, 4 lvs, concurrent pvscans
-# (attempting to have the pvscans run concurrently and race
-# to activate the VG)
-
-vgcreate $vg3 "$dev1" "$dev2" "$dev3"
-lvcreate -l1 -an -n $lv1 $vg3 "$dev1"
-lvcreate -l1 -an -n $lv2 $vg3 "$dev2"
-lvcreate -l1 -an -n $lv3 $vg3 "$dev3"
-lvcreate -l8 -an -n $lv4 -i 2 $vg3 "$dev1" "$dev2"
-
-PVID1=$(pvs "$dev1" --noheading -o uuid | tr -d - | awk '{print $1}')
-PVID2=$(pvs "$dev2" --noheading -o uuid | tr -d - | awk '{print $1}')
-PVID3=$(pvs "$dev3" --noheading -o uuid | tr -d - | awk '{print $1}')
-
-_clear_online_files
-
-udevadm trigger -c add /sys/block/$BDEV1 &
-udevadm trigger -c add /sys/block/$BDEV2 &
-udevadm trigger -c add /sys/block/$BDEV3
-
-aux udev_wait
-wait_lvm_activate $vg3
-
-ls "$RUNDIR/lvm/pvs_online/$PVID1"
-ls "$RUNDIR/lvm/pvs_online/$PVID2"
-ls "$RUNDIR/lvm/pvs_online/$PVID3"
-ls "$RUNDIR/lvm/vgs_online/$vg3"
-journalctl -u lvm-activate-$vg3 | tee out || true
-grep "now active" out
-check lv_field $vg3/$lv1 lv_active "active"
-check lv_field $vg3/$lv2 lv_active "active"
-check lv_field $vg3/$lv3 lv_active "active"
-check lv_field $vg3/$lv4 lv_active "active"
-
-vgchange -an $vg3
-vgremove -y $vg3
-
-
-# 3 devs, 1 vg, 4 lvs, concurrent pvscans, metadata on only 1 PV
-
-wipe_all
-rm $DF
-touch $DF
-pvcreate --metadatacopies 0 "$dev1"
-pvcreate --metadatacopies 0 "$dev2"
-pvcreate "$dev3"
-
-vgcreate $vg4 "$dev1" "$dev2" "$dev3"
-lvcreate -l1 -an -n $lv1 $vg4 "$dev1"
-lvcreate -l1 -an -n $lv2 $vg4 "$dev2"
-lvcreate -l1 -an -n $lv3 $vg4 "$dev3"
-lvcreate -l8 -an -n $lv4 -i 2 $vg4 "$dev1" "$dev2"
-
-PVID1=$(pvs "$dev1" --noheading -o uuid | tr -d - | awk '{print $1}')
-PVID2=$(pvs "$dev2" --noheading -o uuid | tr -d - | awk '{print $1}')
-PVID3=$(pvs "$dev3" --noheading -o uuid | tr -d - | awk '{print $1}')
-
-_clear_online_files
-
-udevadm trigger -c add /sys/block/$BDEV1 &
-udevadm trigger -c add /sys/block/$BDEV2 &
-udevadm trigger -c add /sys/block/$BDEV3
-
-aux udev_wait
-wait_lvm_activate $vg4
-
-ls "$RUNDIR/lvm/pvs_online/$PVID1"
-ls "$RUNDIR/lvm/pvs_online/$PVID2"
-ls "$RUNDIR/lvm/pvs_online/$PVID3"
-ls "$RUNDIR/lvm/vgs_online/$vg4"
-journalctl -u lvm-activate-$vg4 | tee out || true
-grep "now active" out
-check lv_field $vg4/$lv1 lv_active "active"
-check lv_field $vg4/$lv2 lv_active "active"
-check lv_field $vg4/$lv3 lv_active "active"
-check lv_field $vg4/$lv4 lv_active "active"
-
-vgchange -an $vg4
-vgremove -y $vg4
-
-
-# 3 devs, 3 vgs, 2 lvs in each vg, concurrent pvscans
-
-wipe_all
-rm $DF
-touch $DF
-
-vgcreate $vg5 "$dev1"
-vgcreate $vg6 "$dev2"
-vgcreate $vg7 "$dev3"
-lvcreate -l1 -an -n $lv1 $vg5
-lvcreate -l1 -an -n $lv2 $vg5
-lvcreate -l1 -an -n $lv1 $vg6
-lvcreate -l1 -an -n $lv2 $vg6
-lvcreate -l1 -an -n $lv1 $vg7
-lvcreate -l1 -an -n $lv2 $vg7
-
-_clear_online_files
-
-udevadm trigger -c add /sys/block/$BDEV1 &
-udevadm trigger -c add /sys/block/$BDEV2 &
-udevadm trigger -c add /sys/block/$BDEV3
-
-aux udev_wait
-wait_lvm_activate $vg5
-wait_lvm_activate $vg6
-wait_lvm_activate $vg7
-
-ls "$RUNDIR/lvm/vgs_online/$vg5"
-ls "$RUNDIR/lvm/vgs_online/$vg6"
-ls "$RUNDIR/lvm/vgs_online/$vg7"
-journalctl -u lvm-activate-$vg5 | tee out || true
-grep "now active" out
-journalctl -u lvm-activate-$vg6 | tee out || true
-grep "now active" out
-journalctl -u lvm-activate-$vg7 | tee out || true
-grep "now active" out
-check lv_field $vg5/$lv1 lv_active "active"
-check lv_field $vg5/$lv2 lv_active "active"
-check lv_field $vg6/$lv1 lv_active "active"
-check lv_field $vg6/$lv2 lv_active "active"
-check lv_field $vg7/$lv1 lv_active "active"
-check lv_field $vg7/$lv2 lv_active "active"
-
-vgchange -an $vg5
-vgremove -y $vg5
-vgchange -an $vg6
-vgremove -y $vg6
-vgchange -an $vg7
-vgremove -y $vg7
-
-# 3 devs, 1 vg, 1000 LVs
-
-wipe_all
-rm $DF
-touch $DF
-pvcreate --metadatacopies 0 "$dev1"
-pvcreate "$dev2"
-pvcreate "$dev3"
-vgcreate -s 128K $vg8 "$dev1" "$dev2" "$dev3"
-
-# Number of LVs to create
-TEST_DEVS=1000
-# On low-memory boxes let's not stress too much
-test "$(aux total_mem)" -gt 524288 || TEST_DEVS=256
-
-vgcfgbackup -f data $vg8
-
-# Generate a lot of devices (size of 1 extent)
-awk -v TEST_DEVS=$TEST_DEVS '/^\t\}/ {
-    printf("\t}\n\tlogical_volumes {\n");
-    cnt=0;
-    for (i = 0; i < TEST_DEVS; i++) {
-        printf("\t\tlvol%06d  {\n", i);
-        printf("\t\t\tid = \"%06d-1111-2222-3333-2222-1111-%06d\"\n", i, i);
-        print "\t\t\tstatus = [\"READ\", \"WRITE\", \"VISIBLE\"]";
-        print "\t\t\tsegment_count = 1";
-        print "\t\t\tsegment1 {";
-        print "\t\t\t\tstart_extent = 0";
-        print "\t\t\t\textent_count = 1";
-        print "\t\t\t\ttype = \"striped\"";
-        print "\t\t\t\tstripe_count = 1";
-        print "\t\t\t\tstripes = [";
-        print "\t\t\t\t\t\"pv0\", " cnt++;
-        printf("\t\t\t\t]\n\t\t\t}\n\t\t}\n");
-      }
-  }
-  {print}
-' data >data_new
-
-vgcfgrestore -f data_new $vg8
-
-_clear_online_files
-
-udevadm trigger -c add /sys/block/$BDEV1 &
-udevadm trigger -c add /sys/block/$BDEV2 &
-udevadm trigger -c add /sys/block/$BDEV3
-
-aux udev_wait
-wait_lvm_activate $vg8
-
-ls "$RUNDIR/lvm/vgs_online/$vg8"
-journalctl -u lvm-activate-$vg8 | tee out || true
-grep "now active" out
-
-num_active=$(lvs $vg8 --noheading -o active | grep active | wc -l)
-
-test $num_active -eq $TEST_DEVS
-
-vgchange -an $vg8
-vgremove -y $vg8
-
-# 1 pv on an md dev, 1 vg
-
-wait_md_create() {
-	local md=$1
-
-	while :; do
-		if ! grep "$(basename $md)" /proc/mdstat; then
-			echo "$md not ready"
-			cat /proc/mdstat
-			sleep 2
-		else
-			break
-		fi
-	done
-	echo "$md" > WAIT_MD_DEV
-}
-
-test -f /proc/mdstat && grep -q raid1 /proc/mdstat || \
-       modprobe raid1 || skip
-
-mddev="/dev/md33"
-not grep $mddev /proc/mdstat || skip
-
-wipe_all
-rm $DF
-touch $DF
-
-mdadm --create --metadata=1.0 "$mddev" --level 1 --chunk=64 --raid-devices=2 "$dev1" "$dev2"
-wait_md_create "$mddev"
-vgcreate $vg9 "$mddev"
-
-PVIDMD=`pvs $mddev --noheading -o uuid | tr -d - | awk '{print $1}'`
-BDEVMD=$(basename "$mddev")
-
-lvcreate -l1 -an -n $lv1 $vg9
-lvcreate -l1 -an -n $lv2 $vg9
-
-_clear_online_files
-
-udevadm trigger --settle -c add /sys/block/$BDEVMD
-
-wait_lvm_activate $vg9
-
-ls "$RUNDIR/lvm/vgs_online/$vg9"
-journalctl -u lvm-activate-$vg9 | tee out || true
-grep "now active" out
-check lv_field $vg9/$lv1 lv_active "active"
-check lv_field $vg9/$lv2 lv_active "active"
-
-vgchange -an $vg9
-vgremove -y $vg9
-
-mdadm --stop "$mddev"
-aux udev_wait
-wipe_all
-

test/shell/writecache-cache-blocksize.sh

@@ -222,6 +222,8 @@ vgextend $vg "$dev2"
 lvcreate -n $lv1 -l 8 -an $vg "$dev1"
 lvcreate -n $lv2 -l 4 -an $vg "$dev2"
 lvconvert --yes --type writecache --cachevol $lv2 --cachesettings "block_size=4096" $vg/$lv1
+lvs -o writecacheblocksize $vg/$lv1 |tee out
+grep 4096 out
 lvchange -ay $vg/$lv1
 mkfs.xfs -f "$DM_DEV_DIR/$vg/$lv1" |tee out
 grep "sectsz=4096" out

tools/args.h

@@ -153,7 +153,8 @@ arg(cachesize_ARG, '\0', "cachesize", sizemb_VAL, 0, 0,
     "The size of cache to use.\n")
 
 arg(check_ARG, '\0', "check", 0, 0, 0,
-    "Check the content of the devices file.\n")
+    "Checks the content of the devices file.\n"
+    "Reports incorrect device names or PVIDs for entries.\n")
 
 arg(commandprofile_ARG, '\0', "commandprofile", string_VAL, 0, 0,
     "The command profile to use for command configuration.\n"
@@ -234,8 +235,9 @@ arg(deviceidtype_ARG, '\0', "deviceidtype", string_VAL, 0, 0,
     "then it will override the default type that lvm would use.\n")
 
 arg(devices_ARG, '\0', "devices", pv_VAL, ARG_GROUPABLE, 0,
-    "Devices that the command can use. This option can be repeated\n"
-    "or accepts a comma separated list of devices. This overrides\n"
+    "Restricts the devices that are visible and accessible to the command.\n"
+    "Devices not listed will appear to be missing. This option can be\n"
+    "repeated, or accepts a comma separated list of devices. This overrides\n"
     "the devices file.\n")
 
 arg(devicesfile_ARG, '\0', "devicesfile", string_VAL, 0, 0,
@@ -898,6 +900,15 @@ arg(vdopool_ARG, '\0', "vdopool", lv_VAL, 0, 0,
     "The name of a VDO pool LV.\n"
     "See \\fBlvmvdo\\fP(7) for more information about VDO usage.\n")
 
+arg(vdosettings_ARG, '\0', "vdosettings", string_VAL, ARG_GROUPABLE, 0,
+    "Specifies tunable VDO options for VDO LVs.\n"
+    "Use the form 'option=value' or 'option1=value option2=value', or\n"
+    "repeat --vdosettings for each option being set.\n"
+    "These settings override the default VDO behaviors.\n"
+    "To remove vdosettings and revert to the default\n"
+    "VDO behaviors, use --vdosettings 'default'.\n"
+    "See \\fBlvmvdo\\fP(7) for more information.\n")
+
 arg(version_ARG, '\0', "version", 0, 0, 0,
     "Display version information.\n")
 

tools/command-lines.in

@@ -243,6 +243,7 @@ OO_LVCHANGE_META: --addtag Tag, --deltag Tag,
 --setautoactivation Bool, --errorwhenfull Bool, --discards Discards, --zero Bool,
 --cachemode CacheMode, --cachepolicy String, --cachesettings String,
 --minrecoveryrate SizeKB, --maxrecoveryrate SizeKB,
+--vdosettings String,
 --writebehind Number, --writemostly WriteMostlyPV, --persistent n
 
 # It's unfortunate that activate needs to be optionally allowed here;
@@ -341,7 +342,8 @@ OO_LVCONVERT_CACHE: --cachemetadataformat CacheMetadataFormat,
 --cachesettings String, --zero Bool
 
 OO_LVCONVERT_VDO: --metadataprofile String, --readahead Readahead,
---compression Bool, --deduplication Bool, --zero Bool
+--compression Bool, --deduplication Bool, --vdosettings String,
+--zero Bool
 
 OO_LVCONVERT: --alloc Alloc, --background, --force, --noudevsync
 
@@ -839,7 +841,7 @@ OO_LVCREATE_POOL: --poolmetadatasize SizeMB, --poolmetadataspare Bool, --chunksi
 
 OO_LVCREATE_THINPOOL: --discards Discards, --errorwhenfull Bool
 
-OO_LVCREATE_VDO: --compression Bool, --deduplication Bool
+OO_LVCREATE_VDO: --compression Bool, --deduplication Bool, --vdosettings String
 ---
 
 lvcreate --type error --size SizeMB VG
@@ -1359,10 +1361,10 @@ OO: --aligned, --all, --binary, --colon, --columns,
 --configreport ConfigReport, --foreign, --history, --ignorelockingfailure,
 --logonly, --maps, --noheadings,
 --nosuffix, --options String, --sort String, --readonly,
---reportformat ReportFmt, --segments, --select String, --separator String,
+--segments, --select String, --separator String,
 --shared, --unbuffered, --units Units
 OP: VG|LV|Tag ...
-IO: --partial, --ignoreskippedcluster
+IO: --partial, --ignoreskippedcluster, --reportformat ReportFmt
 ID: lvdisplay_general
 
 ---
@@ -1590,10 +1592,10 @@ pvdisplay
 OO: --aligned, --all, --binary, --colon, --columns, --configreport ConfigReport,
 --foreign, --ignorelockingfailure,
 --logonly, --maps, --noheadings, --nosuffix, --options String,
---readonly, --reportformat ReportFmt, --select String, --separator String, --shared,
+--readonly, --select String, --separator String, --shared,
 --short, --sort String, --unbuffered, --units Units
 OP: PV|Tag ...
-IO: --ignoreskippedcluster
+IO: --ignoreskippedcluster, --reportformat ReportFmt
 ID: pvdisplay_general
 
 ---
@@ -1809,10 +1811,10 @@ vgdisplay
 OO: --activevolumegroups, --aligned, --binary, --colon, --columns,
 --configreport ConfigReport, --foreign, --ignorelockingfailure,
 --logonly, --noheadings, --nosuffix,
---options String, --readonly, --reportformat ReportFmt, --select String,
+--options String, --readonly, --select String,
 --shared, --short, --separator String, --sort String, --unbuffered, --units Units
 OP: VG|Tag ...
-IO: --partial, --ignoreskippedcluster
+IO: --partial, --ignoreskippedcluster, --reportformat ReportFmt
 ID: vgdisplay_general
 
 ---

tools/lvchange.c

@@ -755,6 +755,43 @@ out:
 	return r;
 }
 
+static int _lvchange_vdo(struct cmd_context *cmd,
+			 struct logical_volume *lv,
+			 uint32_t *mr)
+{
+	struct lv_segment *seg;
+	int updated = 0;
+
+	seg = first_seg(lv);
+
+	// With VDO LV given flip to VDO pool
+	if (seg_is_vdo(seg))
+		seg = first_seg(seg_lv(seg, 0));
+
+	if (!get_vdo_settings(cmd, &seg->vdo_params, &updated))
+		return_0;
+
+	if ((updated & VDO_CHANGE_OFFLINE) &&
+	    lv_info(cmd, seg->lv, 1, NULL, 0, 0)) {
+		log_error("Cannot change VDO settings for active VDO pool %s.",
+			  display_lvname(seg->lv));
+		// TODO maybe add --force support with prompt here
+		log_print_unless_silent("VDO pool %s with all its LVs needs to be deactivated.",
+					display_lvname(seg->lv));
+		return 0;
+	}
+
+	if (updated) {
+		if (!dm_vdo_validate_target_params(&seg->vdo_params, 0 /* vdo_size */))
+			return_0;
+
+		/* Request caller to commit and reload metadata */
+		*mr |= MR_RELOAD;
+	}
+
+	return 1;
+}
+
 static int _lvchange_tag(struct cmd_context *cmd, struct logical_volume *lv,
 			 int arg, uint32_t *mr)
 {
@@ -1154,6 +1191,7 @@ static int _option_requires_direct_commit(int opt_enum)
 		cachemode_ARG,
 		cachepolicy_ARG,
 		cachesettings_ARG,
+		vdosettings_ARG,
 		-1
 	};
 
@@ -1354,7 +1392,10 @@ static int _lvchange_properties_single(struct cmd_context *cmd,
 			docmds++;
 			doit += _lvchange_cache(cmd, lv, &mr);
 			break;
-
+		case vdosettings_ARG:
+			docmds++;
+			doit += _lvchange_vdo(cmd, lv, &mr);
+			break;
 		default:
 			log_error(INTERNAL_ERROR "Failed to check for option %s",
 				  arg_long_option_name(i));

tools/lvconvert.c

@@ -5456,13 +5456,8 @@ static int _lvconvert_to_vdopool_single(struct cmd_context *cmd,
 	if (!fill_vdo_target_params(cmd, &vdo_params, &vdo_pool_header_size, vg->profile))
 		goto_out;
 
-	if (arg_is_set(cmd, compression_ARG))
-		vdo_params.use_compression =
-			arg_int_value(cmd, compression_ARG, 0);
-
-	if (arg_is_set(cmd, deduplication_ARG))
-		vdo_params.use_deduplication =
-			arg_int_value(cmd, deduplication_ARG, 0);
+	if (!get_vdo_settings(cmd, &vdo_params, NULL))
+		return_0;
 
 	if (!activate_lv(cmd, lv)) {
 		log_error("Cannot activate %s.", display_lvname(lv));
@@ -5487,7 +5482,9 @@ static int _lvconvert_to_vdopool_single(struct cmd_context *cmd,
 	}
 
 	if (vdo_pool_zero) {
-		if (!wipe_lv(lv, (struct wipe_params) { .do_zero = 1, .do_wipe_signatures = 1,
+		if (test_mode()) {
+			log_verbose("Test mode: Skipping activation, zeroing and signature wiping.");
+		} else if (!wipe_lv(lv, (struct wipe_params) { .do_zero = 1, .do_wipe_signatures = 1,
 			     .yes = arg_count(cmd, yes_ARG),
 			     .force = arg_count(cmd, force_ARG)})) {
 			log_error("Aborting. Failed to wipe VDO data store.");
@@ -6072,6 +6069,69 @@ bad:
 	return 0;
 }
 
+static int _check_writecache_memory(struct cmd_context *cmd, struct logical_volume *lv_fast,
+				     uint32_t block_size_sectors)
+{
+	char line[128];
+	FILE *fp;
+	uint64_t cachevol_size_bytes = lv_fast->size * SECTOR_SIZE;
+	uint64_t need_mem_bytes = 0;
+	uint64_t proc_mem_bytes = 0;
+	uint64_t need_mem_gb;
+	uint64_t proc_mem_gb;
+	unsigned long long proc_mem_kb = 0;
+
+	if (!(fp = fopen("/proc/meminfo", "r")))
+		goto skip_proc;
+
+	while (fgets(line, sizeof(line), fp)) {
+		if (strncmp(line, "MemTotal:", 9))
+			continue;
+		if (sscanf(line, "%*s%llu%*s", &proc_mem_kb) != 1)
+			break;
+		break;
+	}
+	(void)fclose(fp);
+
+	proc_mem_bytes = proc_mem_kb * 1024;
+
+ skip_proc:
+	/* dm-writecache memory consumption per block is 88 bytes */
+	if (block_size_sectors == 8) {
+		need_mem_bytes = cachevol_size_bytes * 88 / 4096;
+	} else if (block_size_sectors == 1) {
+		need_mem_bytes = cachevol_size_bytes * 88 / 512;
+	} else {
+		/* shouldn't happen */
+		log_warn("Unknown memory usage for unknown writecache block_size_sectors %u", block_size_sectors);
+		return 1;
+	}
+
+	need_mem_gb = need_mem_bytes / 1073741824;
+	proc_mem_gb = proc_mem_bytes / 1073741824;
+
+	/*
+	 * warn if writecache needs > 50% of main memory, and
+	 * confirm if writecache needs > 90% of main memory.
+	 */
+	if (need_mem_bytes >= (proc_mem_bytes / 2)) {
+		log_warn("WARNING: writecache size %s will use %llu GiB of system memory (%llu GiB).",
+			  display_size(cmd, lv_fast->size),
+			  (unsigned long long)need_mem_gb,
+			  (unsigned long long)proc_mem_gb);
+
+		if (need_mem_gb >= (proc_mem_gb * 9 / 10)) {
+			if (!arg_is_set(cmd, yes_ARG) &&
+			    yes_no_prompt("Continue adding writecache? [y/n]: ") == 'n') {
+				log_error("Conversion aborted.");
+				return 0;
+			}
+		}
+	}
+
+	return 1;
+}
+
 int lvconvert_writecache_attach_single(struct cmd_context *cmd,
 					struct logical_volume *lv,
 					struct processing_handle *handle)
@@ -6160,6 +6220,12 @@ int lvconvert_writecache_attach_single(struct cmd_context *cmd,
 		goto_bad;
 	}
 
+	if (!_check_writecache_memory(cmd, lv_fast, block_size_sectors)) {
+		if (!is_active && !deactivate_lv(cmd, lv))
+			stack;
+		goto_bad;
+	}
+
 	if (!is_active) {
 		if (!deactivate_lv(cmd, lv)) {
 			log_error("Failed to deactivate LV after checking block size %s", display_lvname(lv));

tools/lvcreate.c

@@ -698,6 +698,23 @@ static int _read_cache_params(struct cmd_context *cmd,
 	return 1;
 }
 
+static int _read_vdo_params(struct cmd_context *cmd,
+			    struct lvcreate_params *lp)
+{
+	if (!seg_is_vdo(lp))
+		return 1;
+
+	// prefiling settings here
+	if (!fill_vdo_target_params(cmd, &lp->vdo_params,  &lp->vdo_pool_header_size, NULL))
+		return_0;
+
+	// override with optional vdo settings
+	if (!get_vdo_settings(cmd, &lp->vdo_params, NULL))
+		return_0;
+
+	return 1;
+}
+
 static int _read_activation_params(struct cmd_context *cmd,
 				   struct volume_group *vg,
 				   struct lvcreate_params *lp)
@@ -824,12 +841,16 @@ static int _lvcreate_params(struct cmd_context *cmd,
 	autobackup_ARG,\
 	available_ARG,\
 	contiguous_ARG,\
+	devices_ARG,\
+	devicesfile_ARG,\
 	ignoreactivationskip_ARG,\
 	ignoremonitoring_ARG,\
+	journal_ARG,\
 	metadataprofile_ARG,\
 	monitor_ARG,\
 	mirrors_ARG,\
 	name_ARG,\
+	nohints_ARG,\
 	noudevsync_ARG,\
 	permission_ARG,\
 	persistent_ARG,\
@@ -884,7 +905,8 @@ static int _lvcreate_params(struct cmd_context *cmd,
 #define VDO_POOL_ARGS \
 	vdopool_ARG,\
 	compression_ARG,\
-	deduplication_ARG
+	deduplication_ARG,\
+	vdosettings_ARG
 
 	/* Cache and cache-pool segment type */
 	if (seg_is_cache(lp)) {
@@ -1094,19 +1116,6 @@ static int _lvcreate_params(struct cmd_context *cmd,
 						zero_ARG,
 						-1))
 			return_0;
-
-		// FIXME: prefiling here - this is wrong place
-		// but will work for this moment
-		if (!fill_vdo_target_params(cmd, &lp->vdo_params, &lp->vdo_pool_header_size, NULL))
-			return_0;
-
-		if (arg_is_set(cmd, compression_ARG))
-			lp->vdo_params.use_compression =
-				arg_int_value(cmd, compression_ARG, 0);
-
-		if (arg_is_set(cmd, deduplication_ARG))
-			lp->vdo_params.use_deduplication =
-				arg_int_value(cmd, deduplication_ARG, 0);
 	}
 
 	/* Check options shared between more segment types */
@@ -1194,6 +1203,7 @@ static int _lvcreate_params(struct cmd_context *cmd,
 			      &lp->pool_metadata_size, &lp->pool_metadata_spare,
 			      &lp->chunk_size, &lp->discards, &lp->zero_new_blocks)) ||
 	    !_read_cache_params(cmd, lp) ||
+	    !_read_vdo_params(cmd, lp) ||
 	    !_read_mirror_and_raid_params(cmd, lp))
 		return_0;
 

tools/lvmcmdline.c

@@ -3058,6 +3058,7 @@ int lvm_run_command(struct cmd_context *cmd, int argc, char **argv)
 	int i;
 	int skip_hyphens;
 	int refresh_done = 0;
+	int io;
 
 	/* Avoid excessive access to /etc/localtime and set TZ variable for glibc
 	 * so it does not need to check /etc/localtime everytime that needs that info */
@@ -3140,6 +3141,20 @@ int lvm_run_command(struct cmd_context *cmd, int argc, char **argv)
 	if (!(cmd->command = _find_command(cmd, cmd->name, &argc, argv)))
 		return EINVALID_CMD_LINE;
 
+	/*
+	 * If option --foo is set which is listed in IO (ignore option) in
+	 * command-lines.in, then unset foo.  Commands won't usually use an
+	 * ignored option, but there can be shared code that checks for --foo,
+	 * and should not find it to be set.
+	 */
+	for (io = 0; io < cmd->command->io_count; io++) {
+		int opt = cmd->command->ignore_opt_args[io].opt;
+		if (arg_is_set(cmd, opt)) {
+			log_debug("Ignore opt %d", opt);
+			cmd->opt_arg_values[opt].count = 0;
+		}
+	}
+
 	/*
 	 * Remaining position args after command name and --options are removed.
 	 */
@@ -3291,6 +3306,7 @@ int lvm_run_command(struct cmd_context *cmd, int argc, char **argv)
 	hints_exit(cmd);
 	lvmcache_destroy(cmd, 1, 1);
 	label_scan_destroy(cmd);
+	devices_file_exit(cmd);
 
 	if ((config_string_cft = remove_config_tree_by_source(cmd, CONFIG_STRING)))
 		dm_config_destroy(config_string_cft);

tools/lvmdevices.c

@@ -112,7 +112,7 @@ static void _search_devs_for_pvids(struct cmd_context *cmd, struct dm_list *sear
 		dev = devl->dev;
 		cmd->filter->wipe(cmd, cmd->filter, dev, NULL);
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, dev, NULL)) {
-			log_warn("WARNING: PVID %s found on %s which is excluded by filter: %s",
+			log_warn("WARNING: PVID %s found on %s which is excluded: %s",
 			 	  dev->pvid, dev_name(dev), dev_filtered_reason(dev));
 			dm_list_del(&devl->list);
 		}
@@ -128,7 +128,6 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 	struct device *dev;
 	struct dev_use *du, *du2;
 	const char *deviceidtype;
-	int changes = 0;
 
 	dm_list_init(&search_pvids);
 	dm_list_init(&found_devs);
@@ -176,13 +175,19 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 		log_error("Failed to read the devices file.");
 		return ECMD_FAILED;
 	}
+
+	prepare_open_file_limit(cmd, dm_list_size(&cmd->use_devices));
+
 	dev_cache_scan(cmd);
 	device_ids_match(cmd);
 
 	if (arg_is_set(cmd, check_ARG) || arg_is_set(cmd, update_ARG)) {
 		int search_count = 0;
+		int update_needed = 0;
 		int invalid = 0;
 
+		unlink_searched_devnames(cmd);
+
 		label_scan_setup_bcache();
 
 		dm_list_iterate_items(du, &cmd->use_devices) {
@@ -222,6 +227,8 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 		 * run just above.
 		 */
 		device_ids_validate(cmd, NULL, &invalid, 1);
+		if (invalid)
+			update_needed = 1;
 
 		/*
 		 * Find and fix any devname entries that have moved to a
@@ -237,33 +244,24 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 				label_scan_invalidate(du->dev);
 		}
 
-		/*
-		 * check du->part
-		 */
-		dm_list_iterate_items(du, &cmd->use_devices) {
-			int part = 0;
-			if (!du->dev)
-				continue;
-			dev = du->dev;
-
-			dev_get_partition_number(dev, &part);
-
-			if (part != du->part) {
-				log_warn("Device %s partition %u has incorrect PART in devices file (%u)",
-					 dev_name(dev), part, du->part);
-				du->part = part;
-				changes++;
-			}
-		}
-
 		if (arg_is_set(cmd, update_ARG)) {
-			if (invalid || !dm_list_empty(&found_devs)) {
+			if (update_needed || !dm_list_empty(&found_devs)) {
 				if (!device_ids_write(cmd))
 					goto_bad;
 				log_print("Updated devices file to version %s", devices_file_version());
 			} else {
 				log_print("No update for devices file is needed.");
 			}
+		} else {
+			/*
+			 * --check exits with an error if the devices file
+			 * needs updates, i.e. running --update would make
+			 * changes.
+			 */
+			if (update_needed) {
+				log_error("Updates needed for devices file.");
+				goto bad;
+			}
 		}
 		goto out;
 	}
@@ -312,7 +310,7 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 		cmd->filter_deviceid_skip = 1;
 
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, dev, NULL)) {
-			log_warn("WARNING: adding device %s that is excluded by filter: %s.",
+			log_warn("WARNING: adding device %s that is excluded: %s.",
 				 dev_name(dev), dev_filtered_reason(dev));
 		}
 
@@ -385,28 +383,27 @@ int lvmdevices(struct cmd_context *cmd, int argc, char **argv)
 		 * No filter because we always want to allow removing a device
 		 * by name from the devices file.
 		 */
-		if (!(dev = dev_cache_get(cmd, devname, NULL))) {
-			log_error("No device found for %s.", devname);
-			goto bad;
-		}
-
-		/*
-		 * dev_cache_scan uses sysfs to check if an LV is using each dev
-		 * and sets this flag is so.
-		 */
-		if (dev_is_used_by_active_lv(cmd, dev, NULL, NULL, NULL, NULL)) {
-			if (!arg_count(cmd, yes_ARG) &&
-			    yes_no_prompt("Device %s is used by an active LV, continue to remove? ", devname) == 'n') {
-				log_error("Device not removed.");
-				goto bad;
+		if ((dev = dev_cache_get(cmd, devname, NULL))) {
+			/*
+			 * dev_cache_scan uses sysfs to check if an LV is using each dev
+			 * and sets this flag is so.
+			 */
+			if (dev_is_used_by_active_lv(cmd, dev, NULL, NULL, NULL, NULL)) {
+				if (!arg_count(cmd, yes_ARG) &&
+			    	    yes_no_prompt("Device %s is used by an active LV, continue to remove? ", devname) == 'n') {
+					log_error("Device not removed.");
+					goto bad;
+				}
 			}
+			if ((du = get_du_for_dev(cmd, dev)))
+				goto dev_del;
 		}
 
-		if (!(du = get_du_for_dev(cmd, dev))) {
-			log_error("Device not found in devices file.");
+		if (!(du = get_du_for_devname(cmd, devname))) {
+			log_error("No devices file entry for %s.", devname);
 			goto bad;
 		}
-
+ dev_del:
 		dm_list_del(&du->list);
 		free_du(du);
 		device_ids_write(cmd);

tools/pvcreate.c

@@ -144,7 +144,8 @@ int pvcreate(struct cmd_context *cmd, int argc, char **argv)
 
 	cmd->create_edit_devices_file = 1;
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_ECMD_FAILED;
 
 	if (!(handle = init_processing_handle(cmd, NULL))) {
 		log_error("Failed to initialize processing handle.");

tools/pvremove.c

@@ -45,7 +45,8 @@ int pvremove(struct cmd_context *cmd, int argc, char **argv)
 
 	clear_hint_file(cmd);
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_ECMD_FAILED;
 
 	/* When forcibly clearing a PV we don't care about a VG lock. */
 	if (pp.force == DONT_PROMPT_OVERRIDE)

tools/pvscan.c

@@ -857,21 +857,11 @@ static int _get_devs_from_saved_vg(struct cmd_context *cmd, const char *vgname,
 
 		devno = MKDEV(file_major, file_minor);
 
-		if (!setup_devno_in_dev_cache(cmd, devno)) {
-			log_error_pvscan(cmd, "No device set up for %d:%d PVID %s", file_major, file_minor, pvid);
-			goto bad;
-		}
-
-		if (!(dev = dev_cache_get_by_devt(cmd, devno, NULL, NULL))) {
+		if (!(dev = dev_cache_get_by_devt(cmd, devno))) {
 			log_error_pvscan(cmd, "No device found for %d:%d PVID %s", file_major, file_minor, pvid);
 			goto bad;
 		}
 
-		/*
-		 * Do not need to match device_id here, see comment after
-		 * get_devs_from_saved_vg about relying on pvid online file.
-		 */
-
 		name1 = dev_name(dev);
 		name2 = pvl->pv->device_hint;
 
@@ -1109,11 +1099,17 @@ static int _pvscan_aa(struct cmd_context *cmd, struct pvscan_aa_params *pp,
 		 * PROCESS_SKIP_SCAN: we have already done lvmcache_label_scan
 		 * so tell process_each to skip it.
 		 */
+		if (do_all)
+			read_flags |= PROCESS_SKIP_SCAN;
 
+		/*
+		 * When the command is processing specific devs (not all), it
+		 * has done setup_devices_no_file_match() to avoid matching ids
+		 * fo all devs unnecessarily, but now that we're falling back
+		 * to process_each_vg() we need to complete the id matching.
+		 */
 		if (!do_all)
-			lvmcache_label_scan(cmd);
-
-		read_flags |= PROCESS_SKIP_SCAN;
+			device_ids_match(cmd);
 
 		ret = process_each_vg(cmd, 0, NULL, NULL, vgnames, read_flags, 0, handle, _pvscan_aa_single);
 	}
@@ -1196,15 +1192,11 @@ static int _get_args_devs(struct cmd_context *cmd, struct dm_list *pvscan_args,
 	/* in common usage, no dev will be found for a devno */
 
 	dm_list_iterate_items(arg, pvscan_args) {
-		if (arg->devname) {
-			if (!setup_devname_in_dev_cache(cmd, arg->devname))
-				log_error_pvscan(cmd, "No device set up for name arg %s", arg->devname);
+		if (arg->devname)
 			arg->dev = dev_cache_get(cmd, arg->devname, NULL);
-		} else if (arg->devno) {
-			if (!setup_devno_in_dev_cache(cmd, arg->devno))
-				log_error_pvscan(cmd, "No device set up for devno arg %d", (int)arg->devno);
-			arg->dev = dev_cache_get_by_devt(cmd, arg->devno, NULL, NULL);
-		} else
+		else if (arg->devno)
+			arg->dev = dev_cache_get_by_devt(cmd, arg->devno);
+		else
 			return_0;
 	}
 
@@ -1265,7 +1257,7 @@ static void _set_pv_devices_online(struct cmd_context *cmd, struct volume_group
 
 		devno = MKDEV(major, minor);
 
-		if (!(dev = dev_cache_get_by_devt(cmd, devno, NULL, NULL))) {
+		if (!(dev = dev_cache_get_by_devt(cmd, devno))) {
 			log_print_pvscan(cmd, "VG %s PV %s no device found for %d:%d",
 					 vg->name, pvid, major, minor);
 			pvl->pv->status |= MISSING_PV;
@@ -1362,7 +1354,7 @@ static int _online_devs(struct cmd_context *cmd, int do_all, struct dm_list *pvs
 		devsize = dev->size;
 		if (!devsize &&
 		    !dev_get_size(dev, &devsize)) {
-			log_print("pvscan[%d] PV %s can get device size.", getpid(), dev_name(dev));
+			log_print_pvscan(cmd, "PV %s missing device size.", dev_name(dev));
 			release_vg(vg);
 			continue;
 		}
@@ -1634,7 +1626,8 @@ static int _pvscan_cache_all(struct cmd_context *cmd, int argc, char **argv,
 	 * which we want 'pvscan --cache' to do, and that uses
 	 * info from lvmcache, e.g. duplicate pv info.
 	 */
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_0;
 
 	cmd->pvscan_recreate_hints = 0;
 	cmd->use_hints = 0;
@@ -1680,13 +1673,11 @@ static int _pvscan_cache_args(struct cmd_context *cmd, int argc, char **argv,
 	cmd->pvscan_cache_single = 1;
 
 	/*
-	 * Special pvscan-specific setup steps to avoid looking
-	 * at any devices except for device args.
-	 * Read devices file and determine if devices file will be used.
-	 * Does not do dev_cache_scan (adds nothing to dev-cache), and
-	 * does not do any device id matching.
+	 * "no_file_match" means that when the devices file is used,
+	 * setup_devices will skip matching devs to devices file entries.
+	 * Specific devs must be matched later with device_ids_match_dev().
 	 */
-	if (!setup_devices_for_pvscan_cache(cmd)) {
+	if (!setup_devices_no_file_match(cmd)) {
 		log_error_pvscan(cmd, "Failed to set up devices.");
 		return 0;
 	}
@@ -1745,21 +1736,17 @@ static int _pvscan_cache_args(struct cmd_context *cmd, int argc, char **argv,
 	log_debug("pvscan_cache_args: filter devs nodata");
 
 	/*
-	 * Match dev args with the devices file because special/optimized
-	 * device setup was used above which does not check the devices file.
-	 * If a match fails here do not exclude it, that will be done below by
-	 * passes_filter() which runs filter-deviceid. The
-	 * relax_deviceid_filter case needs to be able to work around
+	 * Match dev args with the devices file because
+	 * setup_devices_no_file_match() was used above which skipped checking
+	 * the devices file.  If a match fails here do not exclude it, that
+	 * will be done below by passes_filter() which runs filter-deviceid.
+	 * The relax_deviceid_filter case needs to be able to work around
 	 * unmatching devs.
 	 */
-
 	if (cmd->enable_devices_file) {
-		dm_list_iterate_items(devl, &pvscan_devs)
+		dm_list_iterate_items_safe(devl, devl2, &pvscan_devs)
 			device_ids_match_dev(cmd, devl->dev);
-
 	}
-	if (cmd->enable_devices_list)
-		device_ids_match_device_list(cmd);
 
 	if (cmd->enable_devices_file && device_ids_use_devname(cmd)) {
 		relax_deviceid_filter = 1;
@@ -1770,7 +1757,7 @@ static int _pvscan_cache_args(struct cmd_context *cmd, int argc, char **argv,
 
 	dm_list_iterate_items_safe(devl, devl2, &pvscan_devs) {
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, devl->dev, NULL)) {
-			log_print_pvscan(cmd, "%s excluded by filters: %s.",
+			log_print_pvscan(cmd, "%s excluded: %s.",
 					 dev_name(devl->dev), dev_filtered_reason(devl->dev));
 			dm_list_del(&devl->list);
 		}
@@ -1800,7 +1787,7 @@ static int _pvscan_cache_args(struct cmd_context *cmd, int argc, char **argv,
 		int has_pvid;
 
 		if (!label_read_pvid(devl->dev, &has_pvid)) {
-			log_print("pvscan[%d] %s cannot read.", getpid(), dev_name(devl->dev));
+			log_print_pvscan(cmd, "%s cannot read label.", dev_name(devl->dev));
 			dm_list_del(&devl->list);
 			continue;
 		}
@@ -1827,7 +1814,7 @@ static int _pvscan_cache_args(struct cmd_context *cmd, int argc, char **argv,
 
 		/* Applies all filters, including those that need data from dev. */
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, devl->dev, NULL)) {
-			log_print_pvscan(cmd, "%s excluded by filters: %s.",
+			log_print_pvscan(cmd, "%s excluded: %s.",
 					 dev_name(devl->dev), dev_filtered_reason(devl->dev));
 			dm_list_del(&devl->list);
 		}

tools/toollib.c

@@ -1192,6 +1192,183 @@ out:
 	return ok;
 }
 
+/*
+ * Compare VDO option name, skip any '_' in name
+ * and also allow to use it without  vdo_[use_] prefix
+ */
+static int _compare_vdo_option(const char *b1, const char *b2)
+{
+	if (strncasecmp(b1, "vdo", 3) == 0) // skip vdo prefix
+		b1 += 3;
+
+	if ((tolower(*b1) != tolower(*b2)) &&
+	    (strncmp(b2, "use_", 4) == 0))
+		b2 += 4;  // try again with skipped prefix 'use_'
+
+	while (*b1 && *b2) {
+		if (tolower(*b1) == tolower(*b2)) {
+			++b1;
+			++b2;
+			continue;	// matching char
+		}
+
+		if (*b1 == '_')
+			++b1;           // skip to next char
+		else if (*b2 == '_')
+			++b2;           // skip to next char
+		else
+			break;          // mismatch
+	}
+
+	return (*b1 || *b2) ? 0 : 1;
+}
+
+#define CHECK_AND_SET(var, onoff) \
+	option = #var;\
+	if (_compare_vdo_option(cn->key, option)) {\
+		if (is_lvchange || !cn->v || (cn->v->type != DM_CFG_INT))\
+			goto err;\
+		if (vtp->var != cn->v->v.i) {\
+			vtp->var = cn->v->v.i;\
+			u |= onoff;\
+		}\
+		continue;\
+	}
+
+#define DO_OFFLINE(var) \
+	CHECK_AND_SET(var, VDO_CHANGE_OFFLINE)
+
+#define DO_ONLINE(var) \
+	CHECK_AND_SET(var, VDO_CHANGE_ONLINE)
+
+int get_vdo_settings(struct cmd_context *cmd,
+		     struct dm_vdo_target_params *vtp,
+		     int *updated)
+{
+	const char *str, *option = NULL;
+	struct arg_value_group_list *group;
+	struct dm_config_tree *result = NULL, *prev = NULL, *current = NULL;
+	struct dm_config_node *cn;
+	int r = 0, u = 0, is_lvchange;
+	int use_compression = vtp->use_compression;
+	int use_deduplication = vtp->use_deduplication;
+	int checked_lvchange;
+
+	if (updated)
+		*updated = 0;
+
+	// Group all --vdosettings
+	dm_list_iterate_items(group, &cmd->arg_value_groups) {
+		if (!grouped_arg_is_set(group->arg_values, vdosettings_ARG))
+			continue;
+
+		if (!(current = dm_config_create()))
+			goto_out;
+		if (prev)
+			current->cascade = prev;
+		prev = current;
+
+		if (!(str = grouped_arg_str_value(group->arg_values,
+						  vdosettings_ARG,
+						  NULL)))
+			goto_out;
+
+		if (!dm_config_parse_without_dup_node_check(current, str, str + strlen(str)))
+			goto_out;
+	}
+
+	if (current) {
+		if (!(result = dm_config_flatten(current)))
+			goto_out;
+
+		checked_lvchange = !strcmp(cmd->name, "lvchange");
+
+		/* Use all acceptable VDO options */
+		for (cn = result->root; cn; cn = cn->sib) {
+			is_lvchange = 0;
+			DO_OFFLINE(ack_threads);
+			DO_OFFLINE(bio_rotation);
+			DO_OFFLINE(bio_threads);
+			DO_OFFLINE(block_map_cache_size_mb);
+			DO_OFFLINE(block_map_era_length);
+			DO_OFFLINE(block_map_period); // alias for block_map_era_length
+			DO_OFFLINE(cpu_threads);
+			DO_OFFLINE(hash_zone_threads);
+			DO_OFFLINE(logical_threads);
+			DO_OFFLINE(max_discard);
+			DO_OFFLINE(physical_threads);
+
+			// Support also these - even when we have regular opts for them
+			DO_ONLINE(use_compression);
+			DO_ONLINE(use_deduplication);
+
+			// Settings bellow cannot be changed with lvchange command
+			is_lvchange = checked_lvchange;
+
+			DO_OFFLINE(check_point_frequency);
+			DO_OFFLINE(index_memory_size_mb);
+			DO_OFFLINE(minimum_io_size);
+			DO_OFFLINE(slab_size_mb);
+			DO_OFFLINE(use_metadata_hints);
+			DO_OFFLINE(use_sparse_index);
+
+			option = "write_policy";
+			if (_compare_vdo_option(cn->key, option)) {
+				if (is_lvchange || !cn->v || (cn->v->type != DM_CFG_STRING))
+					goto err;
+				if (!set_vdo_write_policy(&vtp->write_policy, cn->v->v.str))
+					goto_out;
+				u |= VDO_CHANGE_OFFLINE;
+				continue;
+			}
+
+			log_error("Unknown VDO setting \"%s\".", cn->key);
+			goto out;
+		}
+	}
+
+	if (arg_is_set(cmd, compression_ARG)) {
+		vtp->use_compression = arg_int_value(cmd, compression_ARG, 0);
+		if (vtp->use_compression != use_compression)
+			u |= VDO_CHANGE_ONLINE;
+	}
+
+	if (arg_is_set(cmd, deduplication_ARG)) {
+		vtp->use_deduplication = arg_int_value(cmd, deduplication_ARG, 0);
+		if (vtp->use_deduplication != use_deduplication)
+			u |= VDO_CHANGE_ONLINE;
+	}
+
+	if (updated) {
+		// validation of updated VDO option
+		if (!dm_vdo_validate_target_params(vtp, 0 /* vdo_size */)) {
+err:
+			if (is_lvchange)
+				log_error("Cannot change VDO setting \"vdo_%s\" in existing VDO pool.",
+					  option);
+			else
+				log_error("Invalid argument for VDO setting \"vdo_%s\".",
+					  option);
+			goto out;
+		}
+
+		*updated = u;
+	}
+
+	r = 1;
+out:
+	if (result)
+		dm_config_destroy(result);
+
+	while (prev) {
+		current = prev->cascade;
+		dm_config_destroy(prev);
+		prev = current;
+	}
+
+	return r;
+}
+
 static int _get_one_writecache_setting(struct cmd_context *cmd, struct writecache_settings *settings,
 				       char *key, char *val, uint32_t *block_size_sectors)
 {
@@ -1424,7 +1601,10 @@ int process_each_label(struct cmd_context *cmd, int argc, char **argv,
 
 	log_set_report_object_type(LOG_REPORT_OBJECT_TYPE_LABEL);
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd)) {
+		ret_max = ECMD_FAILED;
+		goto_out;
+	}
 
 	if (argc) {
 		for (; opt < argc; opt++) {
@@ -1434,7 +1614,7 @@ int process_each_label(struct cmd_context *cmd, int argc, char **argv,
 				goto out;
 			}
 
-			if (!(dev = dev_cache_get(cmd, argv[opt], cmd->filter))) {
+			if (!(dev = dev_cache_get_existing(cmd, argv[opt], cmd->filter))) {
 				log_error("Failed to find device "
 					  "\"%s\".", argv[opt]);
 				ret_max = ECMD_FAILED;
@@ -2204,8 +2384,13 @@ int process_each_vg(struct cmd_context *cmd,
 	 * Scan all devices to populate lvmcache with initial
 	 * list of PVs and VGs.
 	 */
-	if (!(read_flags & PROCESS_SKIP_SCAN))
-		lvmcache_label_scan(cmd);
+	if (!(read_flags & PROCESS_SKIP_SCAN)) {
+		if (!lvmcache_label_scan(cmd)) {
+			ret_max = ECMD_FAILED;
+			goto_out;
+		}
+	}
+
 
 	/*
 	 * A list of all VGs on the system is needed when:
@@ -3215,13 +3400,14 @@ int process_each_lv_in_vg(struct cmd_context *cmd, struct volume_group *vg,
 				process_lv = 1;
 			}
 
-			process_lv = process_lv && select_match_lv(cmd, handle, vg, lvl->lv) && _select_matches(handle);
+			_historical_lv.this_glv = glvl->glv;
+			_historical_lv.name = glvl->glv->historical->name;
+
+			process_lv = process_lv && select_match_lv(cmd, handle, vg, &_historical_lv) && _select_matches(handle);
 
 			if (!process_lv)
 				continue;
 
-			_historical_lv.this_glv = glvl->glv;
-			_historical_lv.name = glvl->glv->historical->name;
 			log_very_verbose("Processing historical LV %s in VG %s.", glvl->glv->historical->name, vg->name);
 
 			ret = process_single_lv(cmd, &_historical_lv, handle);
@@ -3754,7 +3940,10 @@ int process_each_lv(struct cmd_context *cmd,
 	 * Scan all devices to populate lvmcache with initial
 	 * list of PVs and VGs.
 	 */
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd)) {
+		ret_max = ECMD_FAILED;
+		goto_out;
+	}
 
 	/*
 	 * A list of all VGs on the system is needed when:
@@ -3870,7 +4059,7 @@ static int _get_arg_devices(struct cmd_context *cmd,
 			return ECMD_FAILED;
 		}
 
-		if (!(dil->dev = dev_cache_get(cmd, sl->str, cmd->filter))) {
+		if (!(dil->dev = dev_cache_get_existing(cmd, sl->str, cmd->filter))) {
 			log_error("Cannot use %s: %s", sl->str, devname_error_reason(sl->str));
 			ret_max = EINIT_FAILED;
 		} else {
@@ -4390,8 +4579,12 @@ int process_each_pv(struct cmd_context *cmd,
 		goto_out;
 	}
 
-	if (!(read_flags & PROCESS_SKIP_SCAN))
-		lvmcache_label_scan(cmd);
+	if (!(read_flags & PROCESS_SKIP_SCAN)) {
+		if (!lvmcache_label_scan(cmd)) {
+			ret_max = ECMD_FAILED;
+			goto_out;
+		}
+	}
 
 	if (!lvmcache_get_vgnameids(cmd, &all_vgnameids, only_this_vgname, 1)) {
 		ret_max = ret;
@@ -5206,7 +5399,7 @@ int pvcreate_each_device(struct cmd_context *cmd,
 		struct device *dev;
 
 		/* No filter used here */
-		if (!(dev = dev_cache_get(cmd, pd->name, NULL))) {
+		if (!(dev = dev_cache_get_existing(cmd, pd->name, NULL))) {
 			log_error("No device found for %s.", pd->name);
 			dm_list_del(&pd->list);
 			dm_list_add(&pp->arg_fail, &pd->list);

tools/toollib.h

@@ -217,6 +217,12 @@ int get_cache_params(struct cmd_context *cmd,
 		     const char **name,
 		     struct dm_config_tree **settings);
 
+#define VDO_CHANGE_ONLINE  1
+#define VDO_CHANGE_OFFLINE 2
+int get_vdo_settings(struct cmd_context *cmd,
+		     struct dm_vdo_target_params *vtp,
+		     int *updated);
+
 int get_writecache_settings(struct cmd_context *cmd, struct writecache_settings *settings,
                             uint32_t *block_size_sectors);
 

tools/vgcfgrestore.c

@@ -132,7 +132,10 @@ int vgcfgrestore(struct cmd_context *cmd, int argc, char **argv)
 
 	clear_hint_file(cmd);
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd)) {
+		unlock_vg(cmd, NULL, vg_name);
+		return_ECMD_FAILED;
+	}
 
 	cmd->handles_unknown_segments = 1;
 

tools/vgcreate.c

@@ -84,7 +84,10 @@ int vgcreate(struct cmd_context *cmd, int argc, char **argv)
 
 	cmd->create_edit_devices_file = 1;
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd)) {
+		unlock_vg(cmd, NULL, vp_new.vg_name);
+		return_ECMD_FAILED;
+	}
 
 	if (lvmcache_vginfo_from_vgname(vp_new.vg_name, NULL)) {
 		unlock_vg(cmd, NULL, vp_new.vg_name);

tools/vgextend.c

@@ -160,7 +160,8 @@ int vgextend(struct cmd_context *cmd, int argc, char **argv)
 
 	cmd->edit_devices_file = 1;
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_ECMD_FAILED;
 
 	if (!(handle = init_processing_handle(cmd, NULL))) {
 		log_error("Failed to initialize processing handle.");

tools/vgimportclone.c

@@ -311,8 +311,8 @@ int vgimportclone(struct cmd_context *cmd, int argc, char **argv)
 	 */
 	dm_list_iterate_items(devl, &vp.new_devs) {
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, devl->dev, "persistent")) {
-			/* FIXME: print which filter */
-			log_error("Device %s was excluded by filters.", dev_name(devl->dev));
+			log_error("Device %s is excluded: %s.",
+				  dev_name(devl->dev), dev_filtered_reason(devl->dev));
 			goto out;
 		}
 	}

tools/vgimportdevices.c

@@ -57,8 +57,7 @@ static int _vgimportdevices_single(struct cmd_context *cmd,
 	dm_list_iterate_items(pvl, &vg->pvs) {
 		pv = pvl->pv;
 
-		if (!idtypestr && pv->device_id_type)
-			idtypestr = pv->device_id_type;
+		idtypestr = pv->device_id_type;
 
 		memcpy(pvid, &pvl->pv->id.uuid, ID_LEN);
 		device_id_add(cmd, pv->dev, pvid, idtypestr, NULL);
@@ -172,6 +171,17 @@ int vgimportdevices(struct cmd_context *cmd, int argc, char **argv)
 	cmd->filter_regex_with_devices_file = 1;
 	cmd->create_edit_devices_file = 1;
 
+	/*
+	 * This helps a user bootstrap existing shared VGs into the devices
+	 * file. Reading the vg to import devices requires locking, but
+	 * lockstart won't find the vg before it's in the devices file.
+	 * So, allow importing devices without an lvmlockd lock (in a
+	 * a shared vg the vg metadata won't be updated with device ids,
+	 * so the lvmlockd lock isn't protecting vg modification.)
+	 */
+	cmd->lockd_gl_disable = 1;
+	cmd->lockd_vg_disable = 1;
+
 	/*
 	 * For each VG:
 	 * device_id_add() each PV in the VG

tools/vgmerge.c

@@ -72,7 +72,8 @@ static int _vgmerge_single(struct cmd_context *cmd, const char *vg_name_to,
 		return ECMD_FAILED;
 	}
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_ECMD_FAILED;
 
 	if (strcmp(vg_name_to, vg_name_from) > 0)
 		lock_vg_from_first = 1;

tools/vgsplit.c

@@ -559,7 +559,8 @@ int vgsplit(struct cmd_context *cmd, int argc, char **argv)
 		return ECMD_FAILED;
 	}
 
-	lvmcache_label_scan(cmd);
+	if (!lvmcache_label_scan(cmd))
+		return_ECMD_FAILED;
 
 	if (!(vginfo_to = lvmcache_vginfo_from_vgname(vg_name_to, NULL))) {
 		if (!validate_name(vg_name_to)) {

udev/69-dm-lvm.rules.in

@@ -1,87 +0,0 @@
-# Copyright (C) 2012,2021 Red Hat, Inc. All rights reserved.
-#
-# This file is part of LVM.
-#
-# This rule requires blkid to be called on block devices before so only devices
-# used as LVM PVs are processed (ID_FS_TYPE="LVM2_member").
-
-SUBSYSTEM!="block", GOTO="lvm_end"
-(LVM_EXEC_RULE)
-
-ENV{DM_UDEV_DISABLE_OTHER_RULES_FLAG}=="1", GOTO="lvm_end"
-
-# Only process devices already marked as a PV - this requires blkid to be called before.
-ENV{ID_FS_TYPE}!="LVM2_member", GOTO="lvm_end"
-ENV{DM_MULTIPATH_DEVICE_PATH}=="1", GOTO="lvm_end"
-ACTION=="remove", GOTO="lvm_end"
-
-# Create /dev/disk/by-id/lvm-pv-uuid-<PV_UUID> symlink for each PV
-ENV{ID_FS_UUID_ENC}=="?*", SYMLINK+="disk/by-id/lvm-pv-uuid-$env{ID_FS_UUID_ENC}"
-
-# If the PV is a special device listed below, scan only if the device is
-# properly activated. These devices are not usable after an ADD event,
-# but they require an extra setup and they are ready after a CHANGE event.
-# Also support coldplugging with ADD event but only if the device is already
-# properly activated.
-# This logic should be eventually moved to rules where those particular
-# devices are processed primarily (MD and loop).
-
-# DM device:
-KERNEL!="dm-[0-9]*", GOTO="next"
-ENV{DM_UDEV_PRIMARY_SOURCE_FLAG}=="1", ENV{DM_ACTIVATION}=="1", GOTO="lvm_scan"
-GOTO="lvm_end"
-
-# MD device:
-LABEL="next"
-KERNEL!="md[0-9]*", GOTO="next"
-IMPORT{db}="LVM_MD_PV_ACTIVATED"
-ACTION=="add", ENV{LVM_MD_PV_ACTIVATED}=="1", GOTO="lvm_scan"
-ACTION=="change", ENV{LVM_MD_PV_ACTIVATED}!="1", TEST=="md/array_state", ENV{LVM_MD_PV_ACTIVATED}="1", GOTO="lvm_scan"
-ACTION=="add", KERNEL=="md[0-9]*p[0-9]*", GOTO="lvm_scan"
-ENV{LVM_MD_PV_ACTIVATED}!="1", ENV{SYSTEMD_READY}="0"
-GOTO="lvm_end"
-
-# Loop device:
-LABEL="next"
-KERNEL!="loop[0-9]*", GOTO="next"
-ACTION=="add", ENV{LVM_LOOP_PV_ACTIVATED}=="1", GOTO="lvm_scan"
-ACTION=="change", ENV{LVM_LOOP_PV_ACTIVATED}!="1", TEST=="loop/backing_file", ENV{LVM_LOOP_PV_ACTIVATED}="1", GOTO="lvm_scan"
-ENV{LVM_LOOP_PV_ACTIVATED}!="1", ENV{SYSTEMD_READY}="0"
-GOTO="lvm_end"
-
-LABEL="next"
-ACTION!="add", GOTO="lvm_end"
-
-LABEL="lvm_scan"
-
-ENV{SYSTEMD_READY}="1"
-
-# pvscan will check if this device completes a VG,
-# i.e. all PVs in the VG are now present with the
-# arrival of this PV.  If so, it prints to stdout:
-# LVM_VG_NAME_COMPLETE='foo'
-#
-# When the VG is complete it can be activated, so
-# vgchange -aay <vgname> is run.  It is run via
-# systemd since it can take longer to run than
-# udev wants to block when processing rules.
-# (if there are hundreds of LVs to activate,
-# the vgchange can take many seconds.)
-#
-# pvscan only reads the single device specified,
-# and uses temp files under /run/lvm to check if
-# other PVs in the VG are present.
-#
-# If event_activation=0 in lvm.conf, this pvscan
-# (using checkcomplete) will do nothing, so that
-# no event-based autoactivation will be happen.
-#
-# TODO: adjust the output of vgchange -aay so that
-# it's better suited to appearing in the journal.
-
-IMPORT{program}="(LVM_EXEC)/lvm pvscan --cache --listvg --checkcomplete --vgonline --udevoutput --journal=output $env{DEVNAME}"
-ENV{LVM_VG_NAME_COMPLETE}=="?*", RUN+="/usr/bin/systemd-run -r --no-block --property DefaultDependencies=no --unit lvm-activate-$env{LVM_VG_NAME_COMPLETE} lvm vgchange -aay --nohints $env{LVM_VG_NAME_COMPLETE}"
-GOTO="lvm_end"
-
-LABEL="lvm_end"
-

udev/Makefile.in

@@ -18,7 +18,7 @@ top_builddir = @top_builddir@
 include $(top_builddir)/make.tmpl
 
 DM_RULES=10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
-LVM_RULES=11-dm-lvm.rules 69-dm-lvm.rules
+LVM_RULES=11-dm-lvm.rules 69-dm-lvm-metad.rules
 
 DM_DIR=$(shell $(GREP) "\#define DM_DIR" $(top_srcdir)/libdm/misc/dm-ioctl.h | $(AWK) '{print $$3}')