pre-release

In setup/teardown ensure we check for and remove any lvm debug
collection, to prevent FS from filling up.

.gitignore

@@ -16,6 +16,10 @@
 *.sw*
 *~
 
+# gcov files:
+*.gcda
+*.gcno
+
 .export.sym
 .exported_symbols_generated
 .gdb_history
@@ -39,9 +43,11 @@ make.tmpl
 
 coverity/coverity_model.xml
 
-# gcov files:
-*.gcda
-*.gcno
+/libdm/.symver_check
+
+daemons/clvmd
+daemons/dmfilemapd
+daemons/lvmetad/
 
 tools/man-generator
 tools/man-generator.c
@@ -102,6 +108,8 @@ test/api/thin_percent.t
 test/api/vglist.t
 test/api/vgtest.t
 test/lib/aux
+test/lib/cache-mq.profile
+test/lib/cache-smq.profile
 test/lib/check
 test/lib/clvmd
 test/lib/dm-version-expected
@@ -111,6 +119,7 @@ test/lib/dmstats
 test/lib/fail
 test/lib/flavour-ndev-cluster
 test/lib/flavour-ndev-cluster-lvmpolld
+test/lib/flavour-ndev-devicesfile
 test/lib/flavour-ndev-lvmetad
 test/lib/flavour-ndev-lvmetad-lvmpolld
 test/lib/flavour-ndev-lvmpolld
@@ -120,6 +129,7 @@ test/lib/flavour-udev-cluster-lvmpolld
 test/lib/flavour-udev-lvmetad
 test/lib/flavour-udev-lvmetad-lvmpolld
 test/lib/flavour-udev-lvmlockd-dlm
+test/lib/flavour-udev-lvmlockd-idm
 test/lib/flavour-udev-lvmlockd-sanlock
 test/lib/flavour-udev-lvmlockd-test
 test/lib/flavour-udev-lvmpolld
@@ -132,8 +142,11 @@ test/lib/lvm
 test/lib/lvm-wrapper
 test/lib/lvmchange
 test/lib/lvmdbusd.profile
+test/lib/lvmdevices
 test/lib/lvmetad
 test/lib/lvmpolld
+test/lib/lvm_import_vdo
+test/lib/lvm_vdo_wrapper
 test/lib/not
 test/lib/paths
 test/lib/paths-common
@@ -143,5 +156,7 @@ test/lib/test
 test/lib/thin-performance.profile
 test/lib/utils
 test/lib/version-expected
+test/lib/vgimportdevices
+
 test/unit/dmraid_t.c
 test/unit/unit-test

Makefile.in

@@ -47,7 +47,7 @@ include $(top_srcdir)/base/Makefile
 include $(top_srcdir)/device_mapper/Makefile
 include $(top_srcdir)/test/unit/Makefile
 
-lib: libdaemon $(BASE_TARGET) $(DEVICE_MAPPER_TARGET)
+lib: include libdaemon $(BASE_TARGET) $(DEVICE_MAPPER_TARGET)
 daemons: lib libdaemon tools
 scripts: lib
 tools: lib libdaemon
@@ -171,6 +171,7 @@ help:
 	@echo "  lcov-dated		Generate lcov with timedate suffix."
 	@echo "  lcov-reset		Reset lcov counters"
 	@echo "  man			Build man pages."
+	@echo "  print-VARIABLE		Resolve make variable."
 	@echo "  rpm			Build rpm."
 	@echo "  run-unit-test		Run unit tests."
 	@echo "  tags			Generate c/etags."

TESTING

@@ -24,7 +24,7 @@ You MUST disable (or mask) any LVM daemons:
 For running cluster tests, we are using singlenode locking. Pass
 `--with-clvmd=singlenode` to configure.
 
-NOTE: This is useful only for testing, and should not be used in produciton
+NOTE: This is useful only for testing, and should not be used in production
 code.
 
 To run D-Bus daemon tests, existing D-Bus session is required.

VERSION

@@ -1 +1 @@
-2.03.14(2)-git (2021-08-11)
+2.03.19(2) (2023-02-21)

VERSION_DM

@@ -1 +1 @@
-1.02.181-git (2021-08-11)
+1.02.191 (2023-02-21)

WHATS_NEW

@@ -1,5 +1,58 @@
-Version 2.03.14 - 
-==================================
+version 2.03.19 - 21st February 2023
+====================================
+  Configure supports --with-systemd-run executed from udev rules.
+  Enhancement for build with MuslC systemd and non-bash system shells (dash).
+  Do not reset SYSTEMD_READY variable in udev for PVs on MD and loop devices.
+  Ensure udev is processing origin LV before its thick snapshots LVs.
+  Fix and improve runtime memory size detection for VDO volumes.
+
+version 2.03.18 - 22nd December 2022
+====================================
+  Fix issues reported by coverity scan.
+  Fix warning for thin pool overprovisioning on lvextend (2.03.17).
+  Add support for writecache metadata_only and pause_writeback settings.
+  Fix missing error messages in lvmdbusd.
+
+Version 2.03.17 - 10th November 2022
+====================================
+  Add new options (--fs, --fsmode) for FS handling when resizing LVs.
+  Fix 'lvremove -S|--select LV' to not also remove its historical LV right away.
+  Fix lv_active field type to binary so --select and --binary applies properly.
+  Switch to use mallinfo2 and use it only with glibc.
+  Error out in lvm shell if using a cmd argument not supported in the shell.
+  Fix lvm shell's lastlog command to report previous pre-command failures.
+  Extend VDO and VDOPOOL without flushing and locking fs.
+  Add --valuesonly option to lvmconfig to print only values without keys.
+  Updates configure with recent autoconf tooling.
+  Fix lvconvert --test --type vdo-pool execution.
+  Add json_std output format for more JSON standard compliant version of output.
+  Fix vdo_slab_size_mb value for converted VDO volume.
+  Fix many corner cases in device_id, including handling of S/N duplicates.
+  Fix various issues in lvmdbusd.
+
+Version 2.03.16 - 18th May 2022
+===============================
+  Fix segfault when handling selection with historical LVs.
+  Add support --vdosettings with lvcreate, lvconvert, lvchange.
+  Filtering multipath devices respects blacklist setting from multipath
+  configuration.
+  lvmdevices support for removing by device id using --deviceidtype and
+  --deldev.
+  Display writecache block size with lvs -o writecache_block_size.
+  Improve cachesettings description in man lvmcache.
+  Fix lossing of delete message on thin-pool extension.
+
+Version 2.03.15 - 07th February 2022
+====================================
+  Remove service based autoactivation. global/event_activation = 0 is NOOP.
+  Improve support for metadata profiles for --type writecache.
+  Use cache or active DM device when available with new kernels.
+  Introduce function to utilize UUIDs from DM_DEVICE_LIST.
+  Increase some hash table size to better support large device sets.
+
+Version 2.03.14 - 20th October 2021
+===================================
+  Device scanning is skipping directories on different filesystems.
   Print info message with too many or too large archived files.
   Reduce metadata readings during scanning phase.
   Optimize computation of crc32 check sum with multiple PVs.
@@ -7,7 +60,7 @@ Version 2.03.14 -
   Filter out unsupported MQ/SMQ cache policy setting.
   Fix memleak in mpath filter.
   Support newer location for VDO statistics.
-  Add support for VDO async-unsage write policy.
+  Add support for VDO async-unsafe write policy.
   Improve lvm_import_vdo script.
   Support VDO LV with lvcreate -ky.
   Fix lvconvert for VDO LV bigger then 2T.
@@ -148,7 +201,7 @@ Version 2.03.10 - 09th August 2020
 
 Version 2.03.09 - 26th March 2020
 =================================
-  Fix formating of vdopool (vdo_slab_size_mb was smaller by 2 bits).
+  Fix formatting of vdopool (vdo_slab_size_mb was smaller by 2 bits).
   Fix showing of a dm kernel error when uncaching a volume with cachevol.
 
 Version 2.03.08 - 11th February 2020

WHATS_NEW_DM

@@ -1,5 +1,26 @@
-Version 1.02.181 - 
-===================================
+Version 1.02.191 - 21st February 2023
+=====================================
+  Improve parallel creation of /dev/mapper/control device node.
+  Import previous ID_FS_* udev records in 13-dm-disk.rules for suspended DM dev.
+  Remove NAME="mapper/control" rule from 10-dm.rules to avoid udev warnings.
+
+Version 1.02.189 - 22nd December 2022
+=====================================
+  Improve 'dmsetup create' without given table line with new kernels.
+
+Version 1.02.187 - 10th November 2022
+=====================================
+  Add DM_REPORT_GROUP_JSON_STD for more JSON standard compliant output format.
+
+Version 1.02.185 - 18th May 2022
+================================
+
+Version 1.02.183 - 07th February 2022
+=====================================
+  Unmangle UUIDs for DM_DEVICE_LIST ioctl.
+
+Version 1.02.181 - 20th October 2021
+====================================
   Add IMA support with 'dmsetup measure' command.
   Add defines DM_NAME_LIST_FLAG_HAS_UUID, DM_NAME_LIST_FLAG_DOESNT_HAVE_UUID.
   Enhance tracking of activated devices when preloading dm tree.

acinclude.m4

@@ -62,6 +62,24 @@ AC_DEFUN([AC_TRY_LDFLAGS],
     fi
 ])
 
+
+dnl PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE,
+dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+dnl -------------------------------------------
+dnl Since: 0.28
+dnl
+dnl Retrieves the value of the pkg-config variable for the given module.
+AC_DEFUN([PKG_CHECK_VAR],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl
+AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl
+
+_PKG_CONFIG([$1], [variable="][$3]["], [$2])
+AS_VAR_COPY([$1], [pkg_cv_][$1])
+
+AS_VAR_IF([$1], [""], [$5], [$4])dnl
+])dnl PKG_CHECK_VAR
+
+
 # ===========================================================================
 #      http://www.gnu.org/software/autoconf-archive/ax_gcc_builtin.html
 # ===========================================================================

aclocal.m4

@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.16.2 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.5 -*- Autoconf -*-
 
-# Copyright (C) 1996-2020 Free Software Foundation, Inc.
+# Copyright (C) 1996-2021 Free Software Foundation, Inc.
 
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -413,7 +413,7 @@ AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
         [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
 ])dnl PKG_HAVE_DEFINE_WITH_MODULES
 
-# Copyright (C) 1999-2020 Free Software Foundation, Inc.
+# Copyright (C) 1999-2021 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -492,7 +492,7 @@ AC_DEFUN([AM_PATH_PYTHON],
   ])
 
   if test "$PYTHON" = :; then
-  dnl Run any user-specified action, or abort.
+    dnl Run any user-specified action, or abort.
     m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])])
   else
 
@@ -501,27 +501,132 @@ AC_DEFUN([AM_PATH_PYTHON],
   dnl trailing zero was eliminated. So now we output just the major
   dnl and minor version numbers, as numbers. Apparently the tertiary
   dnl version is not of interest.
-
+  dnl
   AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version],
-    [am_cv_python_version=`$PYTHON -c "import sys; print('%u.%u' % sys.version_info[[:2]])"`])
+    [am_cv_python_version=`$PYTHON -c "import sys; print ('%u.%u' % sys.version_info[[:2]])"`])
   AC_SUBST([PYTHON_VERSION], [$am_cv_python_version])
 
-  dnl Use the values of $prefix and $exec_prefix for the corresponding
-  dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX.  These are made
-  dnl distinct variables so they can be overridden if need be.  However,
-  dnl general consensus is that you shouldn't need this ability.
-
-  AC_SUBST([PYTHON_PREFIX], ['${prefix}'])
-  AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}'])
-
-  dnl At times (like when building shared libraries) you may want
+  dnl At times, e.g., when building shared libraries, you may want
   dnl to know which OS platform Python thinks this is.
-
+  dnl
   AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform],
     [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`])
   AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform])
 
-  # Just factor out some code duplication.
+  dnl emacs-page
+  dnl If --with-python-sys-prefix is given, use the values of sys.prefix
+  dnl and sys.exec_prefix for the corresponding values of PYTHON_PREFIX
+  dnl and PYTHON_EXEC_PREFIX. Otherwise, use the GNU ${prefix} and
+  dnl ${exec_prefix} variables.
+  dnl
+  dnl The two are made distinct variables so they can be overridden if
+  dnl need be, although general consensus is that you shouldn't need
+  dnl this separation.
+  dnl
+  dnl Also allow directly setting the prefixes via configure options,
+  dnl overriding any default.
+  dnl
+  if test "x$prefix" = xNONE; then
+    am__usable_prefix=$ac_default_prefix
+  else
+    am__usable_prefix=$prefix
+  fi
+
+  # Allow user to request using sys.* values from Python,
+  # instead of the GNU $prefix values.
+  AC_ARG_WITH([python-sys-prefix],
+  [AS_HELP_STRING([--with-python-sys-prefix],
+                  [use Python's sys.prefix and sys.exec_prefix values])],
+  [am_use_python_sys=:],
+  [am_use_python_sys=false])
+
+  # Allow user to override whatever the default Python prefix is.
+  AC_ARG_WITH([python_prefix],
+  [AS_HELP_STRING([--with-python_prefix],
+                  [override the default PYTHON_PREFIX])],
+  [am_python_prefix_subst=$withval
+   am_cv_python_prefix=$withval
+   AC_MSG_CHECKING([for explicit $am_display_PYTHON prefix])
+   AC_MSG_RESULT([$am_cv_python_prefix])],
+  [
+   if $am_use_python_sys; then
+     # using python sys.prefix value, not GNU
+     AC_CACHE_CHECK([for python default $am_display_PYTHON prefix],
+     [am_cv_python_prefix],
+     [am_cv_python_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.prefix)"`])
+
+     dnl If sys.prefix is a subdir of $prefix, replace the literal value of
+     dnl $prefix with a variable reference so it can be overridden.
+     case $am_cv_python_prefix in
+     $am__usable_prefix*)
+       am__strip_prefix=`echo "$am__usable_prefix" | sed 's|.|.|g'`
+       am_python_prefix_subst=`echo "$am_cv_python_prefix" | sed "s,^$am__strip_prefix,\\${prefix},"`
+       ;;
+     *)
+       am_python_prefix_subst=$am_cv_python_prefix
+       ;;
+     esac
+   else # using GNU prefix value, not python sys.prefix
+     am_python_prefix_subst='${prefix}'
+     am_python_prefix=$am_python_prefix_subst
+     AC_MSG_CHECKING([for GNU default $am_display_PYTHON prefix])
+     AC_MSG_RESULT([$am_python_prefix])
+   fi])
+  # Substituting python_prefix_subst value.
+  AC_SUBST([PYTHON_PREFIX], [$am_python_prefix_subst])
+
+  # emacs-page Now do it all over again for Python exec_prefix, but with yet
+  # another conditional: fall back to regular prefix if that was specified.
+  AC_ARG_WITH([python_exec_prefix],
+  [AS_HELP_STRING([--with-python_exec_prefix],
+                  [override the default PYTHON_EXEC_PREFIX])],
+  [am_python_exec_prefix_subst=$withval
+   am_cv_python_exec_prefix=$withval
+   AC_MSG_CHECKING([for explicit $am_display_PYTHON exec_prefix])
+   AC_MSG_RESULT([$am_cv_python_exec_prefix])],
+  [
+   # no explicit --with-python_exec_prefix, but if
+   # --with-python_prefix was given, use its value for python_exec_prefix too.
+   AS_IF([test -n "$with_python_prefix"],
+   [am_python_exec_prefix_subst=$with_python_prefix
+    am_cv_python_exec_prefix=$with_python_prefix
+    AC_MSG_CHECKING([for python_prefix-given $am_display_PYTHON exec_prefix])
+    AC_MSG_RESULT([$am_cv_python_exec_prefix])],
+   [
+    # Set am__usable_exec_prefix whether using GNU or Python values,
+    # since we use that variable for pyexecdir.
+    if test "x$exec_prefix" = xNONE; then
+      am__usable_exec_prefix=$am__usable_prefix
+    else
+      am__usable_exec_prefix=$exec_prefix
+    fi
+    #
+    if $am_use_python_sys; then # using python sys.exec_prefix, not GNU
+      AC_CACHE_CHECK([for python default $am_display_PYTHON exec_prefix],
+      [am_cv_python_exec_prefix],
+      [am_cv_python_exec_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.exec_prefix)"`])
+      dnl If sys.exec_prefix is a subdir of $exec_prefix, replace the
+      dnl literal value of $exec_prefix with a variable reference so it can
+      dnl be overridden.
+      case $am_cv_python_exec_prefix in
+      $am__usable_exec_prefix*)
+        am__strip_prefix=`echo "$am__usable_exec_prefix" | sed 's|.|.|g'`
+        am_python_exec_prefix_subst=`echo "$am_cv_python_exec_prefix" | sed "s,^$am__strip_prefix,\\${exec_prefix},"`
+        ;;
+      *)
+        am_python_exec_prefix_subst=$am_cv_python_exec_prefix
+        ;;
+     esac
+   else # using GNU $exec_prefix, not python sys.exec_prefix
+     am_python_exec_prefix_subst='${exec_prefix}'
+     am_python_exec_prefix=$am_python_exec_prefix_subst
+     AC_MSG_CHECKING([for GNU default $am_display_PYTHON exec_prefix])
+     AC_MSG_RESULT([$am_python_exec_prefix])
+   fi])])
+  # Substituting python_exec_prefix_subst.
+  AC_SUBST([PYTHON_EXEC_PREFIX], [$am_python_exec_prefix_subst])
+
+  # Factor out some code duplication into this shell variable.
   am_python_setup_sysconfig="\
 import sys
 # Prefer sysconfig over distutils.sysconfig, for better compatibility
@@ -541,96 +646,95 @@ try:
 except ImportError:
     pass"
 
-  dnl Set up 4 directories:
+  dnl emacs-page Set up 4 directories:
 
-  dnl pythondir -- where to install python scripts.  This is the
-  dnl   site-packages directory, not the python standard library
-  dnl   directory like in previous automake betas.  This behavior
-  dnl   is more consistent with lispdir.m4 for example.
+  dnl 1. pythondir: where to install python scripts.  This is the
+  dnl    site-packages directory, not the python standard library
+  dnl    directory like in previous automake betas.  This behavior
+  dnl    is more consistent with lispdir.m4 for example.
   dnl Query distutils for this directory.
-  AC_CACHE_CHECK([for $am_display_PYTHON script directory],
-    [am_cv_python_pythondir],
-    [if test "x$prefix" = xNONE
-     then
-       am_py_prefix=$ac_default_prefix
-     else
-       am_py_prefix=$prefix
-     fi
-     am_cv_python_pythondir=`$PYTHON -c "
+  dnl
+  AC_CACHE_CHECK([for $am_display_PYTHON script directory (pythondir)],
+  [am_cv_python_pythondir],
+  [if test "x$am_cv_python_prefix" = x; then
+     am_py_prefix=$am__usable_prefix
+   else
+     am_py_prefix=$am_cv_python_prefix
+   fi
+   am_cv_python_pythondir=`$PYTHON -c "
 $am_python_setup_sysconfig
 if can_use_sysconfig:
-    sitedir = sysconfig.get_path('purelib', vars={'base':'$am_py_prefix'})
+  sitedir = sysconfig.get_path('purelib', vars={'base':'$am_py_prefix'})
 else:
-    from distutils import sysconfig
-    sitedir = sysconfig.get_python_lib(0, 0, prefix='$am_py_prefix')
+  from distutils import sysconfig
+  sitedir = sysconfig.get_python_lib(0, 0, prefix='$am_py_prefix')
 sys.stdout.write(sitedir)"`
-     case $am_cv_python_pythondir in
-     $am_py_prefix*)
-       am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
-       am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"`
-       ;;
-     *)
-       case $am_py_prefix in
-         /usr|/System*) ;;
-         *)
-	  am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages
-	  ;;
-       esac
-       ;;
+   #
+   case $am_cv_python_pythondir in
+   $am_py_prefix*)
+     am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
+     am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,\\${PYTHON_PREFIX},"`
+     ;;
+   *)
+     case $am_py_prefix in
+       /usr|/System*) ;;
+       *) am_cv_python_pythondir="\${PYTHON_PREFIX}/lib/python$PYTHON_VERSION/site-packages"
+          ;;
      esac
-    ])
+     ;;
+   esac
+  ])
   AC_SUBST([pythondir], [$am_cv_python_pythondir])
 
-  dnl pkgpythondir -- $PACKAGE directory under pythondir.  Was
-  dnl   PYTHON_SITE_PACKAGE in previous betas, but this naming is
-  dnl   more consistent with the rest of automake.
-
+  dnl 2. pkgpythondir: $PACKAGE directory under pythondir.  Was
+  dnl    PYTHON_SITE_PACKAGE in previous betas, but this naming is
+  dnl    more consistent with the rest of automake.
+  dnl
   AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE])
 
-  dnl pyexecdir -- directory for installing python extension modules
-  dnl   (shared libraries)
+  dnl 3. pyexecdir: directory for installing python extension modules
+  dnl    (shared libraries).
   dnl Query distutils for this directory.
-  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory],
-    [am_cv_python_pyexecdir],
-    [if test "x$exec_prefix" = xNONE
-     then
-       am_py_exec_prefix=$am_py_prefix
-     else
-       am_py_exec_prefix=$exec_prefix
-     fi
-     am_cv_python_pyexecdir=`$PYTHON -c "
+  dnl
+  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory (pyexecdir)],
+  [am_cv_python_pyexecdir],
+  [if test "x$am_cv_python_exec_prefix" = x; then
+     am_py_exec_prefix=$am__usable_exec_prefix
+   else
+     am_py_exec_prefix=$am_cv_python_exec_prefix
+   fi
+   am_cv_python_pyexecdir=`$PYTHON -c "
 $am_python_setup_sysconfig
 if can_use_sysconfig:
-    sitedir = sysconfig.get_path('platlib', vars={'platbase':'$am_py_prefix'})
+  sitedir = sysconfig.get_path('platlib', vars={'platbase':'$am_py_exec_prefix'})
 else:
-    from distutils import sysconfig
-    sitedir = sysconfig.get_python_lib(1, 0, prefix='$am_py_prefix')
+  from distutils import sysconfig
+  sitedir = sysconfig.get_python_lib(1, 0, prefix='$am_py_exec_prefix')
 sys.stdout.write(sitedir)"`
-     case $am_cv_python_pyexecdir in
-     $am_py_exec_prefix*)
-       am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
-       am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"`
-       ;;
-     *)
-       case $am_py_exec_prefix in
-         /usr|/System*) ;;
-         *)
-	   am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages
-	   ;;
-       esac
-       ;;
+   #
+   case $am_cv_python_pyexecdir in
+   $am_py_exec_prefix*)
+     am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
+     am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,\\${PYTHON_EXEC_PREFIX},"`
+     ;;
+   *)
+     case $am_py_exec_prefix in
+       /usr|/System*) ;;
+       *) am_cv_python_pyexecdir="\${PYTHON_EXEC_PREFIX}/lib/python$PYTHON_VERSION/site-packages"
+          ;;
      esac
-    ])
+     ;;
+   esac
+  ])
   AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir])
 
-  dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE)
-
+  dnl 4. pkgpyexecdir: $(pyexecdir)/$(PACKAGE)
+  dnl
   AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE])
 
   dnl Run any user-specified action.
   $2
   fi
-
 ])
 
 
@@ -653,7 +757,7 @@ for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]]
 sys.exit(sys.hexversion < minverhex)"
   AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])])
 
-# Copyright (C) 2001-2020 Free Software Foundation, Inc.
+# Copyright (C) 2001-2021 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,

autoconf/install-sh

@@ -1,7 +1,7 @@
-#!/bin/sh
+#!/usr/bin/sh
 # install - install a program, script, or datafile
 
-scriptversion=2006-10-14.15
+scriptversion=2020-11-14.01; # UTC
 
 # This originates from X11R5 (mit/util/scripts/install.sh), which was
 # later released in X11R6 (xc/config/util/install.sh) with the
@@ -35,57 +35,62 @@ scriptversion=2006-10-14.15
 # FSF changes to this file are in the public domain.
 #
 # Calling this script install-sh is preferred over install.sh, to prevent
-# `make' implicit rules from creating a file called install from it
+# 'make' implicit rules from creating a file called install from it
 # when there is no Makefile.
 #
 # This script is compatible with the BSD install script, but was written
 # from scratch.
 
+tab='	'
 nl='
 '
-IFS=" ""	$nl"
+IFS=" $tab$nl"
 
-# set DOITPROG to echo to test this script
+# Set DOITPROG to "echo" to test this script.
 
-# Don't use :- since 4.3BSD and earlier shells don't like it.
-doit="${DOITPROG-}"
-if test -z "$doit"; then
-  doit_exec=exec
-else
-  doit_exec=$doit
-fi
+doit=${DOITPROG-}
+doit_exec=${doit:-exec}
 
 # Put in absolute file names if you don't have them in your path;
 # or use environment vars.
 
-mvprog="${MVPROG-mv}"
-cpprog="${CPPROG-cp}"
-chmodprog="${CHMODPROG-chmod}"
-chownprog="${CHOWNPROG-chown}"
-chgrpprog="${CHGRPPROG-chgrp}"
-stripprog="${STRIPPROG-strip}"
-rmprog="${RMPROG-rm}"
-mkdirprog="${MKDIRPROG-mkdir}"
+chgrpprog=${CHGRPPROG-chgrp}
+chmodprog=${CHMODPROG-chmod}
+chownprog=${CHOWNPROG-chown}
+cmpprog=${CMPPROG-cmp}
+cpprog=${CPPROG-cp}
+mkdirprog=${MKDIRPROG-mkdir}
+mvprog=${MVPROG-mv}
+rmprog=${RMPROG-rm}
+stripprog=${STRIPPROG-strip}
 
-posix_glob=
 posix_mkdir=
 
 # Desired mode of installed file.
 mode=0755
 
+# Create dirs (including intermediate dirs) using mode 755.
+# This is like GNU 'install' as of coreutils 8.32 (2020).
+mkdir_umask=22
+
+backupsuffix=
+chgrpcmd=
 chmodcmd=$chmodprog
 chowncmd=
-chgrpcmd=
-stripcmd=
+mvcmd=$mvprog
 rmcmd="$rmprog -f"
-mvcmd="$mvprog"
+stripcmd=
+
 src=
 dst=
 dir_arg=
-dstarg=
-no_target_directory=
+dst_arg=
 
-usage="Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
+copy_on_change=false
+is_target_a_directory=possibly
+
+usage="\
+Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
    or: $0 [OPTION]... SRCFILES... DIRECTORY
    or: $0 [OPTION]... -t DIRECTORY SRCFILES...
    or: $0 [OPTION]... -d DIRECTORIES...
@@ -95,91 +100,116 @@ In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
 In the 4th, create DIRECTORIES.
 
 Options:
--c         (ignored)
--d         create directories instead of installing files.
--g GROUP   $chgrpprog installed files to GROUP.
--m MODE    $chmodprog installed files to MODE.
--o USER    $chownprog installed files to USER.
--s         $stripprog installed files.
--t DIRECTORY  install into DIRECTORY.
--T         report an error if DSTFILE is a directory.
---help     display this help and exit.
---version  display version info and exit.
+     --help     display this help and exit.
+     --version  display version info and exit.
+
+  -c            (ignored)
+  -C            install only if different (preserve data modification time)
+  -d            create directories instead of installing files.
+  -g GROUP      $chgrpprog installed files to GROUP.
+  -m MODE       $chmodprog installed files to MODE.
+  -o USER       $chownprog installed files to USER.
+  -p            pass -p to $cpprog.
+  -s            $stripprog installed files.
+  -S SUFFIX     attempt to back up existing files, with suffix SUFFIX.
+  -t DIRECTORY  install into DIRECTORY.
+  -T            report an error if DSTFILE is a directory.
 
 Environment variables override the default commands:
-  CHGRPPROG CHMODPROG CHOWNPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG
+  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
+  RMPROG STRIPPROG
+
+By default, rm is invoked with -f; when overridden with RMPROG,
+it's up to you to specify -f if you want it.
+
+If -S is not specified, no backups are attempted.
+
+Email bug reports to bug-automake@gnu.org.
+Automake home page: https://www.gnu.org/software/automake/
 "
 
 while test $# -ne 0; do
   case $1 in
-    -c) shift
-        continue;;
+    -c) ;;
 
-    -d) dir_arg=true
-        shift
-        continue;;
+    -C) copy_on_change=true;;
+
+    -d) dir_arg=true;;
 
     -g) chgrpcmd="$chgrpprog $2"
-        shift
-        shift
-        continue;;
+        shift;;
 
     --help) echo "$usage"; exit $?;;
 
     -m) mode=$2
-        shift
-        shift
-	case $mode in
-	  *' '* | *'	'* | *'
-'*	  | *'*'* | *'?'* | *'['*)
-	    echo "$0: invalid mode: $mode" >&2
-	    exit 1;;
-	esac
-        continue;;
+        case $mode in
+          *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*)
+            echo "$0: invalid mode: $mode" >&2
+            exit 1;;
+        esac
+        shift;;
 
     -o) chowncmd="$chownprog $2"
-        shift
-        shift
-        continue;;
+        shift;;
 
-    -s) stripcmd=$stripprog
-        shift
-        continue;;
+    -p) cpprog="$cpprog -p";;
 
-    -t) dstarg=$2
-	shift
-	shift
-	continue;;
+    -s) stripcmd=$stripprog;;
 
-    -T) no_target_directory=true
-	shift
-	continue;;
+    -S) backupsuffix="$2"
+        shift;;
+
+    -t)
+        is_target_a_directory=always
+        dst_arg=$2
+        # Protect names problematic for 'test' and other utilities.
+        case $dst_arg in
+          -* | [=\(\)!]) dst_arg=./$dst_arg;;
+        esac
+        shift;;
+
+    -T) is_target_a_directory=never;;
 
     --version) echo "$0 $scriptversion"; exit $?;;
 
-    --)	shift
-	break;;
+    --) shift
+        break;;
 
-    -*)	echo "$0: invalid option: $1" >&2
-	exit 1;;
+    -*) echo "$0: invalid option: $1" >&2
+        exit 1;;
 
     *)  break;;
   esac
+  shift
 done
 
-if test $# -ne 0 && test -z "$dir_arg$dstarg"; then
+# We allow the use of options -d and -T together, by making -d
+# take the precedence; this is for compatibility with GNU install.
+
+if test -n "$dir_arg"; then
+  if test -n "$dst_arg"; then
+    echo "$0: target directory not allowed when installing a directory." >&2
+    exit 1
+  fi
+fi
+
+if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
   # When -d is used, all remaining arguments are directories to create.
   # When -t is used, the destination is already specified.
   # Otherwise, the last argument is the destination.  Remove it from $@.
   for arg
   do
-    if test -n "$dstarg"; then
+    if test -n "$dst_arg"; then
       # $@ is not empty: it contains at least $arg.
-      set fnord "$@" "$dstarg"
+      set fnord "$@" "$dst_arg"
       shift # fnord
     fi
     shift # arg
-    dstarg=$arg
+    dst_arg=$arg
+    # Protect names problematic for 'test' and other utilities.
+    case $dst_arg in
+      -* | [=\(\)!]) dst_arg=./$dst_arg;;
+    esac
   done
 fi
 
@@ -188,13 +218,26 @@ if test $# -eq 0; then
     echo "$0: no input file specified." >&2
     exit 1
   fi
-  # It's OK to call `install-sh -d' without argument.
+  # It's OK to call 'install-sh -d' without argument.
   # This can happen when creating conditional directories.
   exit 0
 fi
 
 if test -z "$dir_arg"; then
-  trap '(exit $?); exit' 1 2 13 15
+  if test $# -gt 1 || test "$is_target_a_directory" = always; then
+    if test ! -d "$dst_arg"; then
+      echo "$0: $dst_arg: Is not a directory." >&2
+      exit 1
+    fi
+  fi
+fi
+
+if test -z "$dir_arg"; then
+  do_exit='(exit $ret); exit $ret'
+  trap "ret=129; $do_exit" 1
+  trap "ret=130; $do_exit" 2
+  trap "ret=141; $do_exit" 13
+  trap "ret=143; $do_exit" 15
 
   # Set umask so as not to create temps with too-generous modes.
   # However, 'strip' requires both read and write access to temps.
@@ -205,16 +248,16 @@ if test -z "$dir_arg"; then
 
     *[0-7])
       if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
       else
-	u_plus_rw='% 200'
+        u_plus_rw='% 200'
       fi
       cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
     *)
       if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
       else
-	u_plus_rw=,u+rw
+        u_plus_rw=,u+rw
       fi
       cp_umask=$mode$u_plus_rw;;
   esac
@@ -222,9 +265,9 @@ fi
 
 for src
 do
-  # Protect names starting with `-'.
+  # Protect names problematic for 'test' and other utilities.
   case $src in
-    -*) src=./$src ;;
+    -* | [=\(\)!]) src=./$src;;
   esac
 
   if test -n "$dir_arg"; then
@@ -232,6 +275,10 @@ do
     dstdir=$dst
     test -d "$dstdir"
     dstdir_status=$?
+    # Don't chown directories that already exist.
+    if test $dstdir_status = 0; then
+      chowncmd=""
+    fi
   else
 
     # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
@@ -242,196 +289,154 @@ do
       exit 1
     fi
 
-    if test -z "$dstarg"; then
+    if test -z "$dst_arg"; then
       echo "$0: no destination specified." >&2
       exit 1
     fi
+    dst=$dst_arg
 
-    dst=$dstarg
-    # Protect names starting with `-'.
-    case $dst in
-      -*) dst=./$dst ;;
-    esac
-
-    # If destination is a directory, append the input filename; won't work
-    # if double slashes aren't ignored.
+    # If destination is a directory, append the input filename.
     if test -d "$dst"; then
-      if test -n "$no_target_directory"; then
-	echo "$0: $dstarg: Is a directory" >&2
-	exit 1
+      if test "$is_target_a_directory" = never; then
+        echo "$0: $dst_arg: Is a directory" >&2
+        exit 1
       fi
       dstdir=$dst
-      dst=$dstdir/`basename "$src"`
+      dstbase=`basename "$src"`
+      case $dst in
+	*/) dst=$dst$dstbase;;
+	*)  dst=$dst/$dstbase;;
+      esac
       dstdir_status=0
     else
-      # Prefer dirname, but fall back on a substitute if dirname fails.
-      dstdir=`
-	(dirname "$dst") 2>/dev/null ||
-	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
-	     X"$dst" : 'X\(//\)[^/]' \| \
-	     X"$dst" : 'X\(//\)$' \| \
-	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
-	echo X"$dst" |
-	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)[^/].*/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\).*/{
-		   s//\1/
-		   q
-		 }
-		 s/.*/./; q'
-      `
-
+      dstdir=`dirname "$dst"`
       test -d "$dstdir"
       dstdir_status=$?
     fi
   fi
 
+  case $dstdir in
+    */) dstdirslash=$dstdir;;
+    *)  dstdirslash=$dstdir/;;
+  esac
+
   obsolete_mkdir_used=false
 
   if test $dstdir_status != 0; then
     case $posix_mkdir in
       '')
-	# Create intermediate dirs using mode 755 as modified by the umask.
-	# This is like FreeBSD 'install' as of 1997-10-28.
-	umask=`umask`
-	case $stripcmd.$umask in
-	  # Optimize common cases.
-	  *[2367][2367]) mkdir_umask=$umask;;
-	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+        # With -d, create the new directory with the user-specified mode.
+        # Otherwise, rely on $mkdir_umask.
+        if test -n "$dir_arg"; then
+          mkdir_mode=-m$mode
+        else
+          mkdir_mode=
+        fi
 
-	  *[0-7])
-	    mkdir_umask=`expr $umask + 22 \
-	      - $umask % 100 % 40 + $umask % 20 \
-	      - $umask % 10 % 4 + $umask % 2
-	    `;;
-	  *) mkdir_umask=$umask,go-w;;
-	esac
+        posix_mkdir=false
+	# The $RANDOM variable is not portable (e.g., dash).  Use it
+	# here however when possible just to lower collision chance.
+	tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
 
-	# With -d, create the new directory with the user-specified mode.
-	# Otherwise, rely on $mkdir_umask.
-	if test -n "$dir_arg"; then
-	  mkdir_mode=-m$mode
+	trap '
+	  ret=$?
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null
+	  exit $ret
+	' 0
+
+	# Because "mkdir -p" follows existing symlinks and we likely work
+	# directly in world-writeable /tmp, make sure that the '$tmpdir'
+	# directory is successfully created first before we actually test
+	# 'mkdir -p'.
+	if (umask $mkdir_umask &&
+	    $mkdirprog $mkdir_mode "$tmpdir" &&
+	    exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1
+	then
+	  if test -z "$dir_arg" || {
+	       # Check for POSIX incompatibilities with -m.
+	       # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+	       # other-writable bit of parent directory when it shouldn't.
+	       # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+	       test_tmpdir="$tmpdir/a"
+	       ls_ld_tmpdir=`ls -ld "$test_tmpdir"`
+	       case $ls_ld_tmpdir in
+		 d????-?r-*) different_mode=700;;
+		 d????-?--*) different_mode=755;;
+		 *) false;;
+	       esac &&
+	       $mkdirprog -m$different_mode -p -- "$test_tmpdir" && {
+		 ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"`
+		 test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+	       }
+	     }
+	  then posix_mkdir=:
+	  fi
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir"
 	else
-	  mkdir_mode=
+	  # Remove any dirs left behind by ancient mkdir implementations.
+	  rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null
 	fi
-
-	posix_mkdir=false
-	case $umask in
-	  *[123567][0-7][0-7])
-	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
-	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
-	    ;;
-	  *)
-	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
-	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
-	    if (umask $mkdir_umask &&
-		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
-	    then
-	      if test -z "$dir_arg" || {
-		   # Check for POSIX incompatibilities with -m.
-		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
-		   # other-writeable bit of parent directory when it shouldn't.
-		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
-		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
-		   case $ls_ld_tmpdir in
-		     d????-?r-*) different_mode=700;;
-		     d????-?--*) different_mode=755;;
-		     *) false;;
-		   esac &&
-		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
-		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
-		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
-		   }
-		 }
-	      then posix_mkdir=:
-	      fi
-	      rmdir "$tmpdir/d" "$tmpdir"
-	    else
-	      # Remove any dirs left behind by ancient mkdir implementations.
-	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
-	    fi
-	    trap '' 0;;
-	esac;;
+	trap '' 0;;
     esac
 
     if
       $posix_mkdir && (
-	umask $mkdir_umask &&
-	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+        umask $mkdir_umask &&
+        $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
       )
     then :
     else
 
-      # The umask is ridiculous, or mkdir does not conform to POSIX,
+      # mkdir does not conform to POSIX,
       # or it failed possibly due to a race condition.  Create the
       # directory the slow way, step by step, checking for races as we go.
 
       case $dstdir in
-	/*) prefix=/ ;;
-	-*) prefix=./ ;;
-	*)  prefix= ;;
-      esac
-
-      case $posix_glob in
-        '')
-	  if (set -f) 2>/dev/null; then
-	    posix_glob=true
-	  else
-	    posix_glob=false
-	  fi ;;
+        /*) prefix='/';;
+        [-=\(\)!]*) prefix='./';;
+        *)  prefix='';;
       esac
 
       oIFS=$IFS
       IFS=/
-      $posix_glob && set -f
+      set -f
       set fnord $dstdir
       shift
-      $posix_glob && set +f
+      set +f
       IFS=$oIFS
 
       prefixes=
 
       for d
       do
-	test -z "$d" && continue
+        test X"$d" = X && continue
 
-	prefix=$prefix$d
-	if test -d "$prefix"; then
-	  prefixes=
-	else
-	  if $posix_mkdir; then
-	    (umask=$mkdir_umask &&
-	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
-	    # Don't fail if two instances are running concurrently.
-	    test -d "$prefix" || exit 1
-	  else
-	    case $prefix in
-	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
-	      *) qprefix=$prefix;;
-	    esac
-	    prefixes="$prefixes '$qprefix'"
-	  fi
-	fi
-	prefix=$prefix/
+        prefix=$prefix$d
+        if test -d "$prefix"; then
+          prefixes=
+        else
+          if $posix_mkdir; then
+            (umask $mkdir_umask &&
+             $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+            # Don't fail if two instances are running concurrently.
+            test -d "$prefix" || exit 1
+          else
+            case $prefix in
+              *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+              *) qprefix=$prefix;;
+            esac
+            prefixes="$prefixes '$qprefix'"
+          fi
+        fi
+        prefix=$prefix/
       done
 
       if test -n "$prefixes"; then
-	# Don't fail if two instances are running concurrently.
-	(umask $mkdir_umask &&
-	 eval "\$doit_exec \$mkdirprog $prefixes") ||
-	  test -d "$dstdir" || exit 1
-	obsolete_mkdir_used=true
+        # Don't fail if two instances are running concurrently.
+        (umask $mkdir_umask &&
+         eval "\$doit_exec \$mkdirprog $prefixes") ||
+          test -d "$dstdir" || exit 1
+        obsolete_mkdir_used=true
       fi
     fi
   fi
@@ -444,14 +449,25 @@ do
   else
 
     # Make a couple of temp file names in the proper directory.
-    dsttmp=$dstdir/_inst.$$_
-    rmtmp=$dstdir/_rm.$$_
+    dsttmp=${dstdirslash}_inst.$$_
+    rmtmp=${dstdirslash}_rm.$$_
 
     # Trap to clean up those temp files at exit.
     trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
 
     # Copy the file name to the temp name.
-    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
+    (umask $cp_umask &&
+     { test -z "$stripcmd" || {
+	 # Create $dsttmp read-write so that cp doesn't create it read-only,
+	 # which would cause strip to fail.
+	 if test -z "$doit"; then
+	   : >"$dsttmp" # No need to fork-exec 'touch'.
+	 else
+	   $doit touch "$dsttmp"
+	 fi
+       }
+     } &&
+     $doit_exec $cpprog "$src" "$dsttmp") &&
 
     # and set any options; do chmod last to preserve setuid bits.
     #
@@ -459,49 +475,67 @@ do
     # ignore errors from any of these, just make sure not to ignore
     # errors from the above "$doit $cpprog $src $dsttmp" command.
     #
-    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } \
-      && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } \
-      && { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } \
-      && { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
+    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
+    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
+    { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
+    { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
 
-    # Now rename the file to the real destination.
-    { $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null \
-      || {
-	   # The rename failed, perhaps because mv can't rename something else
-	   # to itself, or perhaps because mv is so ancient that it does not
-	   # support -f.
+    # If -C, don't bother to copy if it wouldn't change the file.
+    if $copy_on_change &&
+       old=`LC_ALL=C ls -dlL "$dst"     2>/dev/null` &&
+       new=`LC_ALL=C ls -dlL "$dsttmp"  2>/dev/null` &&
+       set -f &&
+       set X $old && old=:$2:$4:$5:$6 &&
+       set X $new && new=:$2:$4:$5:$6 &&
+       set +f &&
+       test "$old" = "$new" &&
+       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
+    then
+      rm -f "$dsttmp"
+    else
+      # If $backupsuffix is set, and the file being installed
+      # already exists, attempt a backup.  Don't worry if it fails,
+      # e.g., if mv doesn't support -f.
+      if test -n "$backupsuffix" && test -f "$dst"; then
+        $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null
+      fi
 
-	   # Now remove or move aside any old file at destination location.
-	   # We try this two ways since rm can't unlink itself on some
-	   # systems and the destination file might be busy for other
-	   # reasons.  In this case, the final cleanup might fail but the new
-	   # file should still install successfully.
-	   {
-	     if test -f "$dst"; then
-	       $doit $rmcmd -f "$dst" 2>/dev/null \
-	       || { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null \
-		     && { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }; }\
-	       || {
-		 echo "$0: cannot unlink or rename $dst" >&2
-		 (exit 1); exit 1
-	       }
-	     else
-	       :
-	     fi
-	   } &&
+      # Rename the file to the real destination.
+      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
 
-	   # Now rename the file to the real destination.
-	   $doit $mvcmd "$dsttmp" "$dst"
-	 }
-    } || exit 1
+      # The rename failed, perhaps because mv can't rename something else
+      # to itself, or perhaps because mv is so ancient that it does not
+      # support -f.
+      {
+        # Now remove or move aside any old file at destination location.
+        # We try this two ways since rm can't unlink itself on some
+        # systems and the destination file might be busy for other
+        # reasons.  In this case, the final cleanup might fail but the new
+        # file should still install successfully.
+        {
+          test ! -f "$dst" ||
+          $doit $rmcmd "$dst" 2>/dev/null ||
+          { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+            { $doit $rmcmd "$rmtmp" 2>/dev/null; :; }
+          } ||
+          { echo "$0: cannot unlink or rename $dst" >&2
+            (exit 1); exit 1
+          }
+        } &&
+
+        # Now rename the file to the real destination.
+        $doit $mvcmd "$dsttmp" "$dst"
+      }
+    fi || exit 1
 
     trap '' 0
   fi
 done
 
 # Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
+# eval: (add-hook 'before-save-hook 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-end: "$"
+# time-stamp-time-zone: "UTC0"
+# time-stamp-end: "; # UTC"
 # End:

conf/example.conf.in

@@ -126,7 +126,7 @@ devices {
 	# be used, regardless of this setting, when the --devicesfile
 	# option is set to a specific file name.
 	# This configuration option has an automatic default value.
-	# use_devicesfile = 0
+	# use_devicesfile = @DEFAULT_USE_DEVICES_FILE@
 
 	# Configuration option devices/devicesfile.
 	# The name of the system devices file, listing devices that LVM should use.
@@ -625,13 +625,12 @@ allocation {
 	# Enables or disables whether VDO volume should tag its latency-critical
 	# writes with the REQ_SYNC flag. Some device mapper targets such as dm-raid5
 	# process writes with this flag at a higher priority.
-	# Default is enabled.
 	# This configuration option has an automatic default value.
 	# vdo_use_metadata_hints = 1
 
 	# Configuration option allocation/vdo_minimum_io_size.
 	# The minimum IO size for VDO volume to accept, in bytes.
-	# Valid values are 512 or 4096. The recommended and default value is 4096.
+	# Valid values are 512 or 4096. The recommended value is 4096.
 	# This configuration option has an automatic default value.
 	# vdo_minimum_io_size = 4096
 
@@ -684,7 +683,7 @@ allocation {
 	# Configuration option allocation/vdo_bio_threads.
 	# Specifies the number of threads to use for submitting I/O
 	# operations to the storage device of VDO volume.
-	# The value must be in range [1..100]
+	# The value must be in range [1..100].
 	# Each additional thread after the first will use an additional 18MiB of RAM,
 	# plus 1.12 MiB of RAM per megabyte of configured read cache size.
 	# This configuration option has an automatic default value.
@@ -698,7 +697,7 @@ allocation {
 
 	# Configuration option allocation/vdo_cpu_threads.
 	# Specifies the number of threads to use for CPU-intensive work such as
-	# hashing or compression for VDO volume. The value must be in range [1..100]
+	# hashing or compression for VDO volume. The value must be in range [1..100].
 	# This configuration option has an automatic default value.
 	# vdo_cpu_threads = 2
 
@@ -716,7 +715,7 @@ allocation {
 	# processing based on the hash value computed from the block data.
 	# A logical thread count of 9 or more will require explicitly specifying
 	# a sufficiently large block map cache size, as well.
-	# The value must be in range [0..100].
+	# The value must be in range [0..60].
 	# vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be
 	# either all zero or all non-zero.
 	# This configuration option has an automatic default value.
@@ -758,7 +757,7 @@ allocation {
 	# vdo_max_discard = 1
 
 	# Configuration option allocation/vdo_pool_header_size.
-	# Specified the emptry header size in KiB at the front and end of vdo pool device.
+	# Specified the empty header size in KiB at the front and end of vdo pool device.
 	# This configuration option has an automatic default value.
 	# vdo_pool_header_size = 512
 }
@@ -937,7 +936,7 @@ backup {
 	# archive = 1
 
 	# Configuration option backup/archive_dir.
-	# Location of the metdata archive files.
+	# Location of the metadata archive files.
 	# Remember to back up this directory regularly!
 	# This configuration option has an automatic default value.
 	# archive_dir = "@DEFAULT_SYS_DIR@/@DEFAULT_ARCHIVE_SUBDIR@"
@@ -1151,16 +1150,11 @@ global {
 	# lvdisplay_shows_full_device_path = 0
 
 	# Configuration option global/event_activation.
-	# Activate LVs based on system-generated device events.
-	# When a PV appears on the system, a system-generated uevent triggers
-	# the lvm2-pvscan service which runs the pvscan --cache -aay command.
-	# If the new PV completes a VG, pvscan autoactivates LVs in the VG.
-	# When event_activation is disabled, the lvm2-activation services are
-	# generated and run at fixed points during system startup.  These
-	# services run vgchange -aay to autoactivate LVs in VGs that happen
-	# to be present at that point in time.
-	# See the --setautoactivation option or the auto_activation_volume_list
-	# setting to configure autoactivation for specific VGs or LVs.
+	# Disable event based autoactivation commands.
+	# WARNING: setting this to zero may cause machine startup to fail.
+	# Previously, setting this to zero would enable static autoactivation
+	# services (via the lvm2-activation-generator), but the autoactivation
+	# services and generator have been removed.
 	# This configuration option has an automatic default value.
 	# event_activation = 1
 
@@ -1469,13 +1463,13 @@ activation {
 
 	# Configuration option activation/reserved_stack.
 	# Stack size in KiB to reserve for use while devices are suspended.
-	# Insufficent reserve risks I/O deadlock during device suspension.
+	# Insufficient reserve risks I/O deadlock during device suspension.
 	# This configuration option has an automatic default value.
 	# reserved_stack = 64
 
 	# Configuration option activation/reserved_memory.
 	# Memory size in KiB to reserve for use while devices are suspended.
-	# Insufficent reserve risks I/O deadlock during device suspension.
+	# Insufficient reserve risks I/O deadlock during device suspension.
 	# This configuration option has an automatic default value.
 	# reserved_memory = 8192
 
@@ -1610,7 +1604,7 @@ activation {
 	# This includes LVs that have the following segment types:
 	# raid1, raid4, raid5*, and raid6*.
 	# If a device in the LV fails, the policy determines the steps
-	# performed by dmeventd automatically, and the steps perfomed by the
+	# performed by dmeventd automatically, and the steps performed by the
 	# manual command lvconvert --repair --use-policies.
 	# Automatic handling requires dmeventd to be monitoring the LV.
 	#
@@ -1634,7 +1628,7 @@ activation {
 	# (copies) and a mirror log. A disk log ensures that a mirror LV does
 	# not need to be re-synced (all copies made the same) every time a
 	# machine reboots or crashes. If a device in the LV fails, this policy
-	# determines the steps perfomed by dmeventd automatically, and the steps
+	# determines the steps performed by dmeventd automatically, and the steps
 	# performed by the manual command lvconvert --repair --use-policies.
 	# Automatic handling requires dmeventd to be monitoring the LV.
 	#
@@ -1957,6 +1951,12 @@ activation {
 	#     name for identification.
 	#   json
 	#     JSON format.
+	#   json_std
+	#     JSON format that is more compliant with JSON standard.
+	#     Compared to original "json" format:
+	#       - it does not use double quotes around numeric values,
+	#       - it uses 'null' for undefined numeric values,
+	#       - it prints string list as proper JSON array of strings instead of a single string.
 	# This configuration option has an automatic default value.
 	# output_format = "basic"
 

daemons/cmirrord/cluster.c

@@ -108,7 +108,7 @@ static SaVersionT version = { 'B', 1, 1 };
 #endif
 
 #define DEBUGGING_HISTORY 100
-#define DEBUGGING_BUFLEN 128
+#define DEBUGGING_BUFLEN 270
 #define LOG_SPRINT(cc, f, arg...) do {				\
 		cc->idx++;					\
 		cc->idx = cc->idx % DEBUGGING_HISTORY;		\

daemons/cmirrord/functions.c

@@ -34,7 +34,7 @@
 #define LOG_OFFSET 2
 
 #define RESYNC_HISTORY 50
-#define RESYNC_BUFLEN 128
+#define RESYNC_BUFLEN 270
 //static char resync_history[RESYNC_HISTORY][128];
 //static int idx = 0;
 #define LOG_SPRINT(_lc, f, arg...) do {					\

daemons/dmeventd/libdevmapper-event.c

@@ -928,8 +928,8 @@ void dm_event_log(const char *subsys, int level, const char *file,
 			start = now;
 		now -= start;
 		if (_debug_level)
-			fprintf(stream, "[%2d:%02d] %8x:%-6s%s",
-				(int)now / 60, (int)now % 60,
+			fprintf(stream, "[%2lld:%02lld] %8x:%-6s%s",
+				(long long)now / 60, (long long)now % 60,
 				// TODO: Maybe use shorter ID
 				// ((int)(pthread_self()) >> 6) & 0xffff,
 				(int)pthread_self(), subsys,

daemons/lvmdbusd/automatedproperties.py

@@ -44,10 +44,10 @@ class AutomatedProperties(dbus.service.Object):
 
 	def set_interface(self, interface):
 		"""
-		With inheritance we can't easily tell what interfaces a class provides
+		With inheritance, we can't easily tell what interfaces a class provides,
 		so we will have each class that implements an interface tell the
 		base AutomatedProperties what it is they do provide.  This is kind of
-		clunky and perhaps we can figure out a better way to do this later.
+		clunky, and perhaps we can figure out a better way to do this later.
 		:param interface:       An interface the object supports
 		:return:
 		"""
@@ -88,7 +88,6 @@ class AutomatedProperties(dbus.service.Object):
 			cb, cbe, False)
 		cfg.worker_q.put(r)
 
-
 	@staticmethod
 	def _get_all_prop(obj, interface_name):
 		if interface_name in obj.interface(True):

daemons/lvmdbusd/background.py

@@ -7,16 +7,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-import subprocess
 from . import cfg
 from .cmdhandler import options_to_cli_args, LvmExecutionMeta, call_lvm
 import dbus
-from .utils import pv_range_append, pv_dest_ranges, log_error, log_debug,\
-					mt_async_call
+from .utils import pv_range_append, pv_dest_ranges, log_error, log_debug
 from .request import RequestEntry
 import threading
 import time
-import traceback
 
 
 def pv_move_lv_cmd(move_options, lv_full_name,
@@ -65,18 +62,15 @@ def _move_callback(job_state, line_str):
 def _move_merge(interface_name, command, job_state):
 	# We need to execute these command stand alone by forking & exec'ing
 	# the command always as we will be getting periodic output from them on
-	# the status of the long running operation.
+	# the status of the long-running operation.
 
-	meta = LvmExecutionMeta(time.time(), 0, command, -1000, None, None)
-	cfg.blackbox.add(meta)
+	meta = LvmExecutionMeta(time.time(), 0, command)
+	cfg.flightrecorder.add(meta)
 
 	ec, stdout, stderr = call_lvm(command, line_cb=_move_callback,
 									cb_data=job_state)
-
-	with meta.lock:
-		meta.ended = time.time()
-		meta.ec = ec
-		meta.stderr_txt = stderr
+	ended = time.time()
+	meta.completed(ended, ec, stdout, stderr)
 
 	if ec == 0:
 		job_state.Percent = 100

daemons/lvmdbusd/cfg.py

@@ -16,6 +16,8 @@ from lvmdbusd import path
 
 LVM_CMD = os.getenv('LVM_BINARY', path.LVM_BINARY)
 
+LOCK_FILE = os.getenv("LVM_DBUSD_LOCKFILE", "/var/lock/lvm/lvmdbusd")
+
 # This is the global object manager
 om = None
 
@@ -25,13 +27,13 @@ bus = None
 # Command line args
 args = None
 
-# Set to true if we are depending on external events for updates
+# Set to true if we depend on external events for updates
 got_external_event = False
 
 # Shared state variable across all processes
 run = multiprocessing.Value('i', 1)
 
-# If this is set to true, the current setup support lvm shell and we are
+# If this is set to true, the current setup support lvm shell, and we are
 # running in that mode of operation
 SHELL_IN_USE = None
 
@@ -43,6 +45,9 @@ worker_q = queue.Queue()
 # Main event loop
 loop = None
 
+# Used to instruct the daemon if we should ignore SIGTERM
+ignore_sigterm = False
+
 BUS_NAME = os.getenv('LVM_DBUS_NAME', 'com.redhat.lvmdbus1')
 BASE_INTERFACE = 'com.redhat.lvmdbus1'
 PV_INTERFACE = BASE_INTERFACE + '.Pv'
@@ -90,11 +95,14 @@ vdo_support = False
 db = None
 
 # lvm flight recorder
-blackbox = None
+flightrecorder = None
 
 # RequestEntry ctor
 create_request_entry = None
 
+# Circular debug log
+debug = None
+
 
 def exit_daemon():
     """
@@ -104,3 +112,9 @@ def exit_daemon():
     if run and loop:
         run.value = 0
         loop.quit()
+
+
+# Debug data for lvm
+lvmdebug = None
+
+systemd = False

daemons/lvmdbusd/cmdhandler.py

@@ -6,19 +6,18 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
+import errno
 from subprocess import Popen, PIPE
 import select
 import time
 import threading
 from itertools import chain
 import collections
-import traceback
 import os
 
 from lvmdbusd import cfg
 from lvmdbusd.utils import pv_dest_ranges, log_debug, log_error, add_no_notify,\
-							make_non_block, read_decoded
+			make_non_block, read_decoded, extract_stack_trace, LvmBug, add_config_option, get_error_msg
 from lvmdbusd.lvm_shell_proxy import LVMShellProxy
 
 try:
@@ -26,7 +25,6 @@ try:
 except ImportError:
 	import json
 
-SEP = '{|}'
 
 total_time = 0.0
 total_count = 0
@@ -38,7 +36,7 @@ cmd_lock = threading.RLock()
 
 class LvmExecutionMeta(object):
 
-	def __init__(self, start, ended, cmd, ec, stdout_txt, stderr_txt):
+	def __init__(self, start, ended, cmd, ec=-1000, stdout_txt=None, stderr_txt=None):
 		self.lock = threading.RLock()
 		self.start = start
 		self.ended = ended
@@ -49,32 +47,49 @@ class LvmExecutionMeta(object):
 
 	def __str__(self):
 		with self.lock:
-			return "EC= %d for %s\n" \
-				"STARTED: %f, ENDED: %f\n" \
+			if self.ended == 0:
+				ended_txt = "still running"
+				self.ended = time.time()
+			else:
+				ended_txt = str(time.ctime(self.ended))
+
+			return 'EC= %d for "%s"\n' \
+				"STARTED: %s, ENDED: %s, DURATION: %f\n" \
 				"STDOUT=%s\n" \
 				"STDERR=%s\n" % \
-				(self.ec, str(self.cmd), self.start, self.ended, self.stdout_txt,
-				self.stderr_txt)
+				(self.ec, " ".join(self.cmd), time.ctime(self.start), ended_txt, float(self.ended) - self.start,
+					self.stdout_txt,
+					self.stderr_txt)
+
+	def completed(self, end_time, ec, stdout_txt, stderr_txt):
+		with self.lock:
+			self.ended = end_time
+			self.ec = ec
+			self.stdout_txt = stdout_txt
+			self.stderr_txt = stderr_txt
 
 
 class LvmFlightRecorder(object):
 
 	def __init__(self, size=16):
 		self.queue = collections.deque(maxlen=size)
+		self.lock = threading.RLock()
 
 	def add(self, lvm_exec_meta):
-		self.queue.append(lvm_exec_meta)
+		with self.lock:
+			self.queue.append(lvm_exec_meta)
 
 	def dump(self):
-		with cmd_lock:
+		with self.lock:
 			if len(self.queue):
-				log_error("LVM dbus flight recorder START")
+				log_error("LVM dbus flight recorder START (in order of newest to oldest)")
 				for c in reversed(self.queue):
 					log_error(str(c))
 				log_error("LVM dbus flight recorder END")
+				self.queue.clear()
 
 
-cfg.blackbox = LvmFlightRecorder()
+cfg.flightrecorder = LvmFlightRecorder()
 
 
 def _debug_c(cmd, exit_code, out):
@@ -94,7 +109,7 @@ def call_lvm(command, debug=False, line_cb=None,
 					stdin, CALL MUST EXECUTE QUICKLY and not *block*
 					otherwise call_lvm function will fail to read
 					stdin/stdout.  Return value of call back is ignored
-	:param cb_data: Supplied to callback to allow caller access to
+	:param cb_data: Supplied to call back to allow caller access to
 								its own data
 
 	# Callback signature
@@ -106,6 +121,9 @@ def call_lvm(command, debug=False, line_cb=None,
 	command.insert(0, cfg.LVM_CMD)
 	command = add_no_notify(command)
 
+	# Ensure we get an error message when we fork & exec the lvm command line
+	command = add_config_option(command, "--config", 'log/command_log_selection="log_context!=''"')
+
 	process = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True,
 					env=os.environ)
 
@@ -115,7 +133,7 @@ def call_lvm(command, debug=False, line_cb=None,
 	make_non_block(process.stdout)
 	make_non_block(process.stderr)
 
-	while True:
+	while True and cfg.run.value != 0:
 		try:
 			rd_fd = [process.stdout.fileno(), process.stderr.fileno()]
 			ready = select.select(rd_fd, [], [], 2)
@@ -133,8 +151,8 @@ def call_lvm(command, debug=False, line_cb=None,
 					if i != -1:
 						try:
 							line_cb(cb_data, stdout_text[stdout_index:i])
-						except:
-							st = traceback.format_exc()
+						except BaseException as be:
+							st = extract_stack_trace(be)
 							log_error("call_lvm: line_cb exception: \n %s" % st)
 						stdout_index = i + 1
 					else:
@@ -145,12 +163,31 @@ def call_lvm(command, debug=False, line_cb=None,
 				break
 		except IOError as ioe:
 			log_debug("call_lvm:" + str(ioe))
-			pass
+			break
 
-	if debug or process.returncode != 0:
-		_debug_c(command, process.returncode, (stdout_text, stderr_text))
+	if process.returncode is not None:
+		cfg.lvmdebug.lvm_complete()
+		if debug or (process.returncode != 0 and (process.returncode != 5 and "fullreport" in command)):
+			_debug_c(command, process.returncode, (stdout_text, stderr_text))
+
+		try:
+			report_json = json.loads(stdout_text)
+		except json.decoder.JSONDecodeError:
+			# Some lvm commands don't return json even though we are asking for it to do so.
+			return process.returncode, stdout_text, stderr_text
+
+		error_msg = get_error_msg(report_json)
+		if error_msg:
+			stderr_text += error_msg
+
+		return process.returncode, report_json, stderr_text
+	else:
+		if cfg.run.value == 0:
+			raise SystemExit
+		# We can bail out before the lvm command finished when we get a signal
+		# which is requesting we exit
+		return -errno.EINTR, "", "operation interrupted"
 
-	return process.returncode, stdout_text, stderr_text
 
 # The actual method which gets called to invoke the lvm command, can vary
 # from forking a new process to using lvm shell
@@ -165,18 +202,18 @@ def _shell_cfg():
 		_t_call = lvm_shell.call_lvm
 		cfg.SHELL_IN_USE = lvm_shell
 		return True
-	except Exception:
+	except Exception as e:
 		_t_call = call_lvm
 		cfg.SHELL_IN_USE = None
-		log_error(traceback.format_exc())
-		log_error("Unable to utilize lvm shell, dropping back to fork & exec")
+		log_error("Unable to utilize lvm shell, dropping "
+				  "back to fork & exec\n%s" % extract_stack_trace(e))
 		return False
 
 
 def set_execution(shell):
 	global _t_call
 	with cmd_lock:
-		# If the user requested lvm shell and we are currently setup that
+		# If the user requested lvm shell, and we are currently setup that
 		# way, just return
 		if cfg.SHELL_IN_USE and shell:
 			return True
@@ -200,11 +237,15 @@ def time_wrapper(command, debug=False):
 
 	with cmd_lock:
 		start = time.time()
+		meta = LvmExecutionMeta(start, 0, command)
+		# Add the partial metadata to flight recorder, so if the command hangs
+		# we will see what it was.
+		cfg.flightrecorder.add(meta)
 		results = _t_call(command, debug)
 		ended = time.time()
 		total_time += (ended - start)
 		total_count += 1
-		cfg.blackbox.add(LvmExecutionMeta(start, ended, command, *results))
+		meta.completed(ended, *results)
 	return results
 
 
@@ -214,44 +255,11 @@ call = time_wrapper
 # Default cmd
 # Place default arguments for every command here.
 def _dc(cmd, args):
-	c = [cmd, '--noheading', '--separator', '%s' % SEP, '--nosuffix',
-		'--unbuffered', '--units', 'b']
+	c = [cmd, '--nosuffix', '--unbuffered', '--units', 'b']
 	c.extend(args)
 	return c
 
 
-def parse(out):
-	rc = []
-
-	for line in out.split('\n'):
-		# This line includes separators, so process them
-		if SEP in line:
-			elem = line.split(SEP)
-			cleaned_elem = []
-			for e in elem:
-				e = e.strip()
-				cleaned_elem.append(e)
-
-			if len(cleaned_elem) > 1:
-				rc.append(cleaned_elem)
-		else:
-			t = line.strip()
-			if len(t) > 0:
-				rc.append(t)
-	return rc
-
-
-def parse_column_names(out, column_names):
-	lines = parse(out)
-	rc = []
-
-	for i in range(0, len(lines)):
-		d = dict(list(zip(column_names, lines[i])))
-		rc.append(d)
-
-	return rc
-
-
 def options_to_cli_args(options):
 	rc = []
 	for k, v in list(dict(options).items()):
@@ -613,68 +621,23 @@ def lvm_full_report_json():
 		'--configreport', 'vg', '-o', ','.join(vg_columns),
 		'--configreport', 'lv', '-o', ','.join(lv_columns),
 		'--configreport', 'seg', '-o', ','.join(lv_seg_columns),
-		'--configreport', 'pvseg', '-o', ','.join(pv_seg_columns),
-		'--reportformat', 'json'
+		'--configreport', 'pvseg', '-o', ','.join(pv_seg_columns)
 	])
 
+	# We are running the fullreport command, we will ask lvm to output the debug
+	# data, so we can have the required information for lvm to debug the fullreport failures.
+	# Note: this is disabled by default and can be enabled with env. var.
+	# LVM_DBUSD_COLLECT_LVM_DEBUG=True
+	fn = cfg.lvmdebug.setup()
+	if fn is not None:
+		add_config_option(cmd, "--config", "log {level=7 file=%s syslog=0}" % fn)
+
 	rc, out, err = call(cmd)
 	# When we have an exported vg the exit code of lvs or fullreport will be 5
 	if rc == 0 or rc == 5:
-		# With the current implementation, if we are using the shell then we
-		# are using JSON and JSON is returned back to us as it was parsed to
-		# figure out if we completed OK or not
-		if cfg.SHELL_IN_USE:
-			assert(type(out) == dict)
-			return out
-		else:
-			try:
-				return json.loads(out)
-			except json.decoder.JSONDecodeError as joe:
-				log_error("JSONDecodeError %s, \n JSON=\n%s\n" %
-							(str(joe), out))
-				raise joe
-
-	return None
-
-
-def pv_retrieve_with_segs(device=None):
-	d = []
-	err = ""
-	out = ""
-	rc = 0
-
-	columns = ['pv_name', 'pv_uuid', 'pv_fmt', 'pv_size', 'pv_free',
-				'pv_used', 'dev_size', 'pv_mda_size', 'pv_mda_free',
-				'pv_ba_start', 'pv_ba_size', 'pe_start', 'pv_pe_count',
-				'pv_pe_alloc_count', 'pv_attr', 'pv_tags', 'vg_name',
-				'vg_uuid', 'pvseg_start', 'pvseg_size', 'segtype', 'pv_missing']
-
-	# Lvm has some issues where it returns failure when querying pvs when other
-	# operations are in process, see:
-	# https://bugzilla.redhat.com/show_bug.cgi?id=1274085
-	for i in range(0, 10):
-		cmd = _dc('pvs', ['-o', ','.join(columns)])
-
-		if device:
-			cmd.extend(device)
-
-		rc, out, err = call(cmd)
-
-		if rc == 0:
-			d = parse_column_names(out, columns)
-			break
-		else:
-			time.sleep(0.2)
-			log_debug("LVM Bug workaround, retrying pvs command...")
-
-	if rc != 0:
-		msg = "We were unable to get pvs to return without error after " \
-			"trying 10 times, RC=%d, STDERR=(%s), STDOUT=(%s)" % \
-			(rc, err, out)
-		log_error(msg)
-		raise RuntimeError(msg)
-
-	return d
+		assert(type(out) == dict)
+		return out
+	raise LvmBug("'fullreport' exited with code '%d'" % rc)
 
 
 def pv_resize(device, size_bytes, create_options):
@@ -831,53 +794,6 @@ def activate_deactivate(op, name, activate, control_flags, options):
 	return call(cmd)
 
 
-def vg_retrieve(vg_specific):
-	if vg_specific:
-		assert isinstance(vg_specific, list)
-
-	columns = ['vg_name', 'vg_uuid', 'vg_fmt', 'vg_size', 'vg_free',
-				'vg_sysid', 'vg_extent_size', 'vg_extent_count',
-				'vg_free_count', 'vg_profile', 'max_lv', 'max_pv',
-				'pv_count', 'lv_count', 'snap_count', 'vg_seqno',
-				'vg_mda_count', 'vg_mda_free', 'vg_mda_size',
-				'vg_mda_used_count', 'vg_attr', 'vg_tags']
-
-	cmd = _dc('vgs', ['-o', ','.join(columns)])
-
-	if vg_specific:
-		cmd.extend(vg_specific)
-
-	d = []
-	rc, out, err = call(cmd)
-	if rc == 0:
-		d = parse_column_names(out, columns)
-
-	return d
-
-
-def lv_retrieve_with_segments():
-	columns = ['lv_uuid', 'lv_name', 'lv_path', 'lv_size',
-				'vg_name', 'pool_lv_uuid', 'pool_lv', 'origin_uuid',
-				'origin', 'data_percent',
-				'lv_attr', 'lv_tags', 'vg_uuid', 'lv_active', 'data_lv',
-				'metadata_lv', 'seg_pe_ranges', 'segtype', 'lv_parent',
-				'lv_role', 'lv_layout',
-				'snap_percent', 'metadata_percent', 'copy_percent',
-				'sync_percent', 'lv_metadata_size', 'move_pv', 'move_pv_uuid']
-
-	cmd = _dc('lvs', ['-a', '-o', ','.join(columns)])
-	rc, out, err = call(cmd)
-
-	d = []
-
-	if rc == 0:
-		d = parse_column_names(out, columns)
-
-	return d
-
-
 if __name__ == '__main__':
-	pv_data = pv_retrieve_with_segs()
-
-	for p in pv_data:
-		print(str(p))
+	# Leave this for future debug as needed
+	pass

daemons/lvmdbusd/fetch.py

@@ -6,16 +6,16 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
+import errno
 
 from .pv import load_pvs
 from .vg import load_vgs
 from .lv import load_lvs
 from . import cfg
-from .utils import MThreadRunner, log_debug, log_error
+from .utils import MThreadRunner, log_debug, log_error, LvmBug, extract_stack_trace
 import threading
 import queue
 import time
-import traceback
 
 
 def _main_thread_load(refresh=True, emit_signal=True):
@@ -120,81 +120,108 @@ class StateUpdate(object):
 	@staticmethod
 	def update_thread(obj):
 		exception_count = 0
-
 		queued_requests = []
+
+		def set_results(val):
+			nonlocal queued_requests
+			for idx in queued_requests:
+				idx.set_result(val)
+				# Only clear out the requests after we have given them a result
+				# otherwise we can orphan the waiting threads, and they never
+				# wake up if we get an exception
+				queued_requests = []
+
+		def bailing(rv):
+			set_results(rv)
+			try:
+				while True:
+					item = obj.queue.get(False)
+					item.set_result(rv)
+			except queue.Empty:
+				pass
+
+		def _load_args(requests):
+			"""
+			If we have multiple requests in the queue, they might not all have the same options.  If any of the requests
+			have an option set we need to honor it.
+			"""
+			refresh = any([r.refresh for r in requests])
+			emit_signal = any([r.emit_signal for r in requests])
+			cache_refresh = any([r.cache_refresh for r in requests])
+			log = any([r.log for r in requests])
+			need_main_thread = any([r.need_main_thread for r in requests])
+
+			return refresh, emit_signal, cache_refresh, log, need_main_thread
+
+		def _drain_queue(queued, incoming):
+			try:
+				while True:
+					queued.append(incoming.get(block=False))
+			except queue.Empty:
+				pass
+
+		def _handle_error():
+			nonlocal exception_count
+			exception_count += 1
+
+			if exception_count >= 5:
+				log_error("Too many errors in update_thread, exiting daemon")
+				cfg.debug.dump()
+				cfg.flightrecorder.dump()
+				bailing(errno.EFAULT)
+				cfg.exit_daemon()
+			else:
+				# Slow things down when encountering errors
+				cfg.lvmdebug.complete()
+				time.sleep(1)
+
 		while cfg.run.value != 0:
 			# noinspection PyBroadException
 			try:
-				refresh = True
-				emit_signal = True
-				cache_refresh = True
-				log = True
-				need_main_thread = True
-
 				with obj.lock:
 					wait = not obj.deferred
 					obj.deferred = False
 
 				if len(queued_requests) == 0 and wait:
-					queued_requests.append(obj.queue.get(True, 2))
+					# Note: If we don't have anything for 2 seconds we will
+					# get a queue.Empty exception raised here
+					queued_requests.append(obj.queue.get(block=True, timeout=2))
 
 				# Ok we have one or the deferred queue has some,
-				# check if any others
-				try:
-					while True:
-						queued_requests.append(obj.queue.get(False))
-
-				except queue.Empty:
-					pass
+				# check if any others and grab them too
+				_drain_queue(queued_requests, obj.queue)
 
 				if len(queued_requests) > 1:
 					log_debug("Processing %d updates!" % len(queued_requests),
 							'bg_black', 'fg_light_green')
 
-				# We have what we can, run the update with the needed options
-				for i in queued_requests:
-					if not i.refresh:
-						refresh = False
-					if not i.emit_signal:
-						emit_signal = False
-					if not i.cache_refresh:
-						cache_refresh = False
-					if not i.log:
-						log = False
-					if not i.need_main_thread:
-						need_main_thread = False
-
-				num_changes = load(refresh, emit_signal, cache_refresh, log,
-									need_main_thread)
+				num_changes = load(*_load_args(queued_requests))
 				# Update is done, let everyone know!
-				for i in queued_requests:
-					i.set_result(num_changes)
-
-				# Only clear out the requests after we have given them a result
-				# otherwise we can orphan the waiting threads and they never
-				# wake up if we get an exception
-				queued_requests = []
+				set_results(num_changes)
 
 				# We retrieved OK, clear exception count
 				exception_count = 0
 
 			except queue.Empty:
 				pass
+			except SystemExit:
+				break
+			except LvmBug as bug:
+				# If a lvm bug occurred, we will dump the lvm debug data if
+				# we have it.
+				cfg.lvmdebug.dump()
+				log_error(str(bug))
+				_handle_error()
 			except Exception as e:
-				st = traceback.format_exc()
-				log_error("update_thread exception: \n%s" % st)
-				cfg.blackbox.dump()
-				exception_count += 1
-				if exception_count >= 5:
-					for i in queued_requests:
-						i.set_result(e)
+				log_error("update_thread: \n%s" % extract_stack_trace(e))
+				_handle_error()
+			finally:
+				cfg.lvmdebug.complete()
 
-					log_error("Too many errors in update_thread, exiting daemon")
-					cfg.exit_daemon()
-
-				else:
-					# Slow things down when encountering errors
-					time.sleep(1)
+		# Make sure to unblock any that may be waiting before we exit this thread
+		# otherwise they hang forever ...
+		bailing(Exception("update thread exiting"))
+		log_debug("update thread exiting!")
 
 	def __init__(self):
 		self.lock = threading.RLock()

daemons/lvmdbusd/job.py

@@ -44,7 +44,7 @@ class WaitingClient(object):
 			self.timer_id = GLib.timeout_add_seconds(
 				tmo, WaitingClient._timeout, self)
 
-	# The job finished before the timer popped and we are being notified that
+	# The job finished before the timer popped, and we are being notified that
 	# it's done
 	def notify(self):
 		with self.rlock:
@@ -71,7 +71,7 @@ class JobState(object):
 		self._stderr = ''
 		self._waiting_clients = []
 
-		# This is an lvm command that is just taking too long and doesn't
+		# This is a lvm command that is just taking too long and doesn't
 		# support background operation
 		if self._request:
 			# Faking the percentage when we don't have one
@@ -138,7 +138,7 @@ class JobState(object):
 		# If a waiting client timer pops before the job is done we will allow
 		# the client to remove themselves from the list.  As we have a lock
 		# here and a lock in the waiting client too, and they can be obtained
-		# in different orders, a dead lock can occur.
+		# in different orders, a deadlock can occur.
 		# As this remove is really optional, we will try to acquire the lock
 		# and remove.  If we are unsuccessful it's not fatal, we just delay
 		# the time when the objects can be garbage collected by python
@@ -226,3 +226,21 @@ class Job(AutomatedProperties):
 	def Uuid(self):
 		import uuid
 		return uuid.uuid1()
+
+	# Override the property "getters" implementation for the job interface, so a user can query a job while the queue
+	# is processing items.  Originally all the property get methods were this way, but we changed this in
+	# e53454d6de07de56736303dd2157c3859f6fa848
+
+	# Properties
+	# noinspection PyUnusedLocal
+	@dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE,
+						 in_signature='ss', out_signature='v')
+	def Get(self, interface_name, property_name):
+		# Note: If we get an exception in this handler we won't know about it,
+		# only the side effect of no returned value!
+		return AutomatedProperties._get_prop(self, interface_name, property_name)
+
+	@dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE,
+						 in_signature='s', out_signature='a{sv}')
+	def GetAll(self, interface_name):
+		return AutomatedProperties._get_all_prop(self, interface_name)

daemons/lvmdbusd/loader.py

@@ -42,7 +42,7 @@ def common(retrieve, o_type, search_keys,
 		existing_paths = cfg.om.object_paths_by_type(o_type)
 
 	for o in objects:
-		# Assume we need to add this one to dbus, unless we are refreshing
+		# Assume we need to add this one to dbus, unless we are refreshing,
 		# and it's already present
 		return_object = True
 

daemons/lvmdbusd/lv.py

@@ -10,7 +10,7 @@
 from .automatedproperties import AutomatedProperties
 
 from . import utils
-from .utils import vg_obj_path_generate, log_error, _handle_execute
+from .utils import vg_obj_path_generate, log_error, _handle_execute, LvmBug
 import dbus
 from . import cmdhandler
 from . import cfg
@@ -21,14 +21,12 @@ from .utils import n, n32, d
 from .loader import common
 from .state import State
 from . import background
-from .utils import round_size, mt_remove_dbus_objects
+from .utils import round_size, mt_remove_dbus_objects, lvm_column_key
 from .job import JobState
 
-import traceback
 
-
-# Try and build a key for a LV, so that we sort the LVs with least dependencies
-# first.  This may be error prone because of the flexibility LVM
+# Try and build a key for a LV, so that we sort the LVs with the least dependencies
+# first.  This may be error-prone because of the flexibility LVM
 # provides and what you can stack.
 def get_key(i):
 
@@ -73,67 +71,74 @@ def lvs_state_retrieve(selection, cache_refresh=True):
 	# don't have information available yet.
 	lvs = sorted(cfg.db.fetch_lvs(selection), key=get_key)
 
-	for l in lvs:
-		if cfg.vdo_support:
-			rc.append(LvStateVdo(
-				l['lv_uuid'], l['lv_name'],
-				l['lv_path'], n(l['lv_size']),
-				l['vg_name'],
-				l['vg_uuid'], l['pool_lv_uuid'],
-				l['pool_lv'], l['origin_uuid'], l['origin'],
-				n32(l['data_percent']), l['lv_attr'],
-				l['lv_tags'], l['lv_active'], l['data_lv'],
-				l['metadata_lv'], l['segtype'], l['lv_role'],
-				l['lv_layout'],
-				n32(l['snap_percent']),
-				n32(l['metadata_percent']),
-				n32(l['copy_percent']),
-				n32(l['sync_percent']),
-				n(l['lv_metadata_size']),
-				l['move_pv'],
-				l['move_pv_uuid'],
-				l['vdo_operating_mode'],
-				l['vdo_compression_state'],
-				l['vdo_index_state'],
-				n(l['vdo_used_size']),
-				d(l['vdo_saving_percent']),
-				l['vdo_compression'],
-				l['vdo_deduplication'],
-				l['vdo_use_metadata_hints'],
-				n32(l['vdo_minimum_io_size']),
-				n(l['vdo_block_map_cache_size']),
-				n32(l['vdo_block_map_era_length']),
-				l['vdo_use_sparse_index'],
-				n(l['vdo_index_memory_size']),
-				n(l['vdo_slab_size']),
-				n32(l['vdo_ack_threads']),
-				n32(l['vdo_bio_threads']),
-				n32(l['vdo_bio_rotation']),
-				n32(l['vdo_cpu_threads']),
-				n32(l['vdo_hash_zone_threads']),
-				n32(l['vdo_logical_threads']),
-				n32(l['vdo_physical_threads']),
-				n32(l['vdo_max_discard']),
-				l['vdo_write_policy'],
-				n32(l['vdo_header_size'])))
-		else:
-			rc.append(LvState(
-				l['lv_uuid'], l['lv_name'],
-				l['lv_path'], n(l['lv_size']),
-				l['vg_name'],
-				l['vg_uuid'], l['pool_lv_uuid'],
-				l['pool_lv'], l['origin_uuid'], l['origin'],
-				n32(l['data_percent']), l['lv_attr'],
-				l['lv_tags'], l['lv_active'], l['data_lv'],
-				l['metadata_lv'], l['segtype'], l['lv_role'],
-				l['lv_layout'],
-				n32(l['snap_percent']),
-				n32(l['metadata_percent']),
-				n32(l['copy_percent']),
-				n32(l['sync_percent']),
-				n(l['lv_metadata_size']),
-				l['move_pv'],
-				l['move_pv_uuid']))
+	try:
+		for l in lvs:
+			if cfg.vdo_support:
+				rc.append(LvStateVdo(
+					l['lv_uuid'], l['lv_name'],
+					l['lv_path'], n(l['lv_size']),
+					l['vg_name'],
+					l['vg_uuid'], l['pool_lv_uuid'],
+					l['pool_lv'], l['origin_uuid'], l['origin'],
+					n32(l['data_percent']), l['lv_attr'],
+					l['lv_tags'], l['lv_active'], l['data_lv'],
+					l['metadata_lv'], l['segtype'], l['lv_role'],
+					l['lv_layout'],
+					n32(l['snap_percent']),
+					n32(l['metadata_percent']),
+					n32(l['copy_percent']),
+					n32(l['sync_percent']),
+					n(l['lv_metadata_size']),
+					l['move_pv'],
+					l['move_pv_uuid'],
+					l['vdo_operating_mode'],
+					l['vdo_compression_state'],
+					l['vdo_index_state'],
+					n(l['vdo_used_size']),
+					d(l['vdo_saving_percent']),
+					l['vdo_compression'],
+					l['vdo_deduplication'],
+					l['vdo_use_metadata_hints'],
+					n32(l['vdo_minimum_io_size']),
+					n(l['vdo_block_map_cache_size']),
+					n32(l['vdo_block_map_era_length']),
+					l['vdo_use_sparse_index'],
+					n(l['vdo_index_memory_size']),
+					n(l['vdo_slab_size']),
+					n32(l['vdo_ack_threads']),
+					n32(l['vdo_bio_threads']),
+					n32(l['vdo_bio_rotation']),
+					n32(l['vdo_cpu_threads']),
+					n32(l['vdo_hash_zone_threads']),
+					n32(l['vdo_logical_threads']),
+					n32(l['vdo_physical_threads']),
+					n32(l['vdo_max_discard']),
+					l['vdo_write_policy'],
+					n32(l['vdo_header_size'])))
+			else:
+				rc.append(LvState(
+					l['lv_uuid'], l['lv_name'],
+					l['lv_path'], n(l['lv_size']),
+					l['vg_name'],
+					l['vg_uuid'], l['pool_lv_uuid'],
+					l['pool_lv'], l['origin_uuid'], l['origin'],
+					n32(l['data_percent']), l['lv_attr'],
+					l['lv_tags'], l['lv_active'], l['data_lv'],
+					l['metadata_lv'], l['segtype'], l['lv_role'],
+					l['lv_layout'],
+					n32(l['snap_percent']),
+					n32(l['metadata_percent']),
+					n32(l['copy_percent']),
+					n32(l['sync_percent']),
+					n(l['lv_metadata_size']),
+					l['move_pv'],
+					l['move_pv_uuid']))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 
@@ -274,15 +279,15 @@ class LvStateVdo(LvState):
 					MetaDataPercent, CopyPercent, SyncPercent,
 					MetaDataSizeBytes, move_pv, move_pv_uuid,
 					vdo_operating_mode, vdo_compression_state, vdo_index_state,
-					vdo_used_size,vdo_saving_percent,vdo_compression,
-					vdo_deduplication,vdo_use_metadata_hints,
-					vdo_minimum_io_size,vdo_block_map_cache_size,
-					vdo_block_map_era_length,vdo_use_sparse_index,
-					vdo_index_memory_size,vdo_slab_size,vdo_ack_threads,
-					vdo_bio_threads,vdo_bio_rotation,vdo_cpu_threads,
-					vdo_hash_zone_threads,vdo_logical_threads,
-					vdo_physical_threads,vdo_max_discard,
-					vdo_write_policy,vdo_header_size):
+					vdo_used_size, vdo_saving_percent, vdo_compression,
+					vdo_deduplication, vdo_use_metadata_hints,
+					vdo_minimum_io_size, vdo_block_map_cache_size,
+					vdo_block_map_era_length, vdo_use_sparse_index,
+					vdo_index_memory_size, vdo_slab_size, vdo_ack_threads,
+					vdo_bio_threads, vdo_bio_rotation, vdo_cpu_threads,
+					vdo_hash_zone_threads, vdo_logical_threads,
+					vdo_physical_threads, vdo_max_discard,
+					vdo_write_policy, vdo_header_size):
 		super(LvStateVdo, self).__init__(Uuid, Name, Path, SizeBytes,
 					vg_name, vg_uuid, pool_lv_uuid, PoolLv,
 					origin_uuid, OriginLv, DataPercent, Attr, Tags, active,
@@ -371,8 +376,8 @@ class LvCommon(AutomatedProperties):
 			return dbus.Struct((self.state.Attr[index],
 				type_map.get(self.state.Attr[index], default)),
 								signature="(ss)")
-		except BaseException:
-			st = traceback.format_exc()
+		except BaseException as b:
+			st = utils.extract_stack_trace(b)
 			log_error("attr_struct: \n%s" % st)
 			return dbus.Struct(('?', 'Unavailable'), signature="(ss)")
 
@@ -595,7 +600,7 @@ class Lv(LvCommon):
 				optional_size = space + 512 - remainder
 
 		LvCommon.handle_execute(*cmdhandler.vg_lv_snapshot(
-			lv_name, snapshot_options,name, optional_size))
+			lv_name, snapshot_options, name, optional_size))
 		full_name = "%s/%s" % (dbo.vg_name_lookup(), name)
 		return cfg.om.get_object_path_by_lvm_id(full_name)
 
@@ -635,7 +640,7 @@ class Lv(LvCommon):
 
 		size_change = new_size_bytes - dbo.SizeBytes
 		LvCommon.handle_execute(*cmdhandler.lv_resize(
-			dbo.lvm_id, size_change,pv_dests, resize_options))
+			dbo.lvm_id, size_change, pv_dests, resize_options))
 		return "/"
 
 	@dbus.service.method(
@@ -744,7 +749,7 @@ class Lv(LvCommon):
 		cfg.worker_q.put(r)
 
 	@staticmethod
-	def _writecache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
+	def _caching_common(method, lv_uuid, lv_name, lv_object_path, cache_options):
 		# Make sure we have a dbus object representing it
 		dbo = LvCommon.validate_dbus_object(lv_uuid, lv_name)
 
@@ -753,7 +758,7 @@ class Lv(LvCommon):
 
 		if lv_to_cache:
 			fcn = lv_to_cache.lv_full_name()
-			rc, out, err = cmdhandler.lv_writecache_lv(
+			rc, out, err = method(
 				dbo.lv_full_name(), fcn, cache_options)
 			if rc == 0:
 				# When we cache an LV, the cache pool and the lv that is getting
@@ -770,9 +775,14 @@ class Lv(LvCommon):
 		else:
 			raise dbus.exceptions.DBusException(
 				LV_INTERFACE, 'LV to cache with object path %s not present!' %
-				lv_object_path)
+							  lv_object_path)
 		return lv_converted
 
+	@staticmethod
+	def _writecache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
+		return Lv._caching_common(cmdhandler.lv_writecache_lv, lv_uuid,
+									lv_name, lv_object_path, cache_options)
+
 	@dbus.service.method(
 		dbus_interface=LV_INTERFACE,
 		in_signature='oia{sv}',
@@ -845,10 +855,10 @@ class LvVdoPool(Lv):
 		cfg.worker_q.put(r)
 
 	@dbus.service.method(
-	dbus_interface=VDO_POOL_INTERFACE,
-	in_signature='ia{sv}',
-	out_signature='o',
-	async_callbacks=('cb', 'cbe'))
+		dbus_interface=VDO_POOL_INTERFACE,
+		in_signature='ia{sv}',
+		out_signature='o',
+		async_callbacks=('cb', 'cbe'))
 	def DisableCompression(self, tmo, comp_options, cb, cbe):
 		r = RequestEntry(
 			tmo, LvVdoPool._enable_disable_compression,
@@ -878,10 +888,10 @@ class LvVdoPool(Lv):
 		cfg.worker_q.put(r)
 
 	@dbus.service.method(
-	dbus_interface=VDO_POOL_INTERFACE,
-	in_signature='ia{sv}',
-	out_signature='o',
-	async_callbacks=('cb', 'cbe'))
+		dbus_interface=VDO_POOL_INTERFACE,
+		in_signature='ia{sv}',
+		out_signature='o',
+		async_callbacks=('cb', 'cbe'))
 	def DisableDeduplication(self, tmo, dedup_options, cb, cbe):
 		r = RequestEntry(
 			tmo, LvVdoPool._enable_disable_deduplication,
@@ -952,33 +962,8 @@ class LvCachePool(Lv):
 
 	@staticmethod
 	def _cache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
-		# Make sure we have a dbus object representing cache pool
-		dbo = LvCommon.validate_dbus_object(lv_uuid, lv_name)
-
-		# Make sure we have dbus object representing lv to cache
-		lv_to_cache = cfg.om.get_object_by_path(lv_object_path)
-
-		if lv_to_cache:
-			fcn = lv_to_cache.lv_full_name()
-			rc, out, err = cmdhandler.lv_cache_lv(
-				dbo.lv_full_name(), fcn, cache_options)
-			if rc == 0:
-				# When we cache an LV, the cache pool and the lv that is getting
-				# cached need to be removed from the object manager and
-				# re-created as their interfaces have changed!
-				mt_remove_dbus_objects((dbo, lv_to_cache))
-				cfg.load()
-
-				lv_converted = cfg.om.get_object_path_by_lvm_id(fcn)
-			else:
-				raise dbus.exceptions.DBusException(
-					LV_INTERFACE,
-					'Exit code %s, stderr = %s' % (str(rc), err))
-		else:
-			raise dbus.exceptions.DBusException(
-				LV_INTERFACE, 'LV to cache with object path %s not present!' %
-				lv_object_path)
-		return lv_converted
+		return Lv._caching_common(cmdhandler.lv_cache_lv, lv_uuid, lv_name,
+									lv_object_path, cache_options)
 
 	@dbus.service.method(
 		dbus_interface=CACHE_POOL_INTERFACE,

daemons/lvmdbusd/lvm_shell_proxy.py.in

@@ -14,12 +14,11 @@
 import subprocess
 import shlex
 import os
-import traceback
+import pty
 import sys
 import tempfile
 import time
 import select
-import copy
 
 try:
 	import simplejson as json
@@ -27,9 +26,9 @@ except ImportError:
 	import json
 
 
-from lvmdbusd.cfg import LVM_CMD
+import lvmdbusd.cfg as cfg
 from lvmdbusd.utils import log_debug, log_error, add_no_notify, make_non_block,\
-							read_decoded
+			read_decoded, extract_stack_trace, LvmBug, get_error_msg
 
 SHELL_PROMPT = "lvm> "
 
@@ -43,10 +42,11 @@ def _quote_arg(arg):
 
 class LVMShellProxy(object):
 
-	# Read until we get prompt back and a result
-	# @param: no_output	Caller expects no output to report FD
-	# Returns stdout, report, stderr (report is JSON!)
-	def _read_until_prompt(self, no_output=False):
+	# Read REPORT FD until we have a complete and valid JSON record or give
+	# up trying to get one.
+	#
+	# Returns stdout, report (JSON), stderr
+	def _read_response(self, no_output=False):
 		stdout = ""
 		report = ""
 		stderr = ""
@@ -58,24 +58,27 @@ class LVMShellProxy(object):
 		# Try reading from all FDs to prevent one from filling up and causing
 		# a hang.  Keep reading until we get the prompt back and the report
 		# FD does not contain valid JSON
-		while keep_reading:
+
+		while keep_reading and cfg.run.value != 0:
 			try:
 				rd_fd = [
-					self.lvm_shell.stdout.fileno(),
+					self.parent_stdout_fd,
 					self.report_stream.fileno(),
-					self.lvm_shell.stderr.fileno()]
+					self.parent_stderr_fd]
 				ready = select.select(rd_fd, [], [], 2)
 
 				for r in ready[0]:
-					if r == self.lvm_shell.stdout.fileno():
-						stdout += read_decoded(self.lvm_shell.stdout)
+					if r == self.parent_stdout_fd:
+						for line in self.parent_stdout.readlines():
+							stdout += line
 					elif r == self.report_stream.fileno():
 						report += read_decoded(self.report_stream)
-					elif r == self.lvm_shell.stderr.fileno():
-						stderr += read_decoded(self.lvm_shell.stderr)
+					elif r == self.parent_stderr_fd:
+						for line in self.parent_stderr.readlines():
+							stderr += line
 
 				# Check to see if the lvm process died on us
-				if self.lvm_shell.poll():
+				if self.lvm_shell.poll() is not None:
 					raise Exception(self.lvm_shell.returncode, "%s" % stderr)
 
 				if stdout.endswith(SHELL_PROMPT):
@@ -99,91 +102,96 @@ class LVMShellProxy(object):
 							extra_passes -= 1
 							if extra_passes <= 0:
 								if len(report):
-									raise ValueError("Invalid json: %s" %
+									raise LvmBug("Invalid json: %s" %
 														report)
 								else:
-									raise ValueError(
+									raise LvmBug(
 										"lvm returned no JSON output!")
 
 			except IOError as ioe:
 				log_debug(str(ioe))
-				pass
+				self.exit_shell()
+				raise ioe
+
+		if keep_reading and cfg.run.value == 0:
+			# We didn't complete as we are shutting down
+			# Try to clean up lvm shell process
+			log_debug("exiting lvm shell as we are shutting down")
+			self.exit_shell()
+			raise SystemExit
 
 		return stdout, report_json, stderr
 
 	def _write_cmd(self, cmd):
-		cmd_bytes = bytes(cmd, "utf-8")
-		num_written = self.lvm_shell.stdin.write(cmd_bytes)
-		assert (num_written == len(cmd_bytes))
-		self.lvm_shell.stdin.flush()
+		self.parent_stdin.write(cmd)
+		self.parent_stdin.flush()
 
 	def __init__(self):
-
 		# Create a temp directory
 		tmp_dir = tempfile.mkdtemp(prefix="lvmdbus_")
 		tmp_file = "%s/lvmdbus_report" % (tmp_dir)
 
-		try:
-			# Lets create fifo for the report output
-			os.mkfifo(tmp_file, 0o600)
-		except FileExistsError:
-			pass
+		# Create a fifo for the report output
+		os.mkfifo(tmp_file, 0o600)
 
-		# We have to open non-blocking as the other side isn't open until
-		# we actually fork the process.
+		# Open the fifo for use to read and for lvm child process to write to.
 		self.report_fd = os.open(tmp_file, os.O_NONBLOCK)
 		self.report_stream = os.fdopen(self.report_fd, 'rb', 0)
+		lvm_fd = os.open(tmp_file, os.O_WRONLY)
 
-		# Setup the environment for using our own socket for reporting
-		local_env = copy.deepcopy(os.environ)
-		local_env["LVM_REPORT_FD"] = "32"
-		local_env["LVM_COMMAND_PROFILE"] = "lvmdbusd"
+		# Set up the environment for using our own socket for reporting and disable the abort
+		# logic if lvm logs too much, which easily happens when utilizing the lvm shell.
+		local_env = {"LC_ALL": "C", "LVM_REPORT_FD": "%s" % lvm_fd, "LVM_COMMAND_PROFILE": "lvmdbusd",
+					 "LVM_LOG_FILE_MAX_LINES": "0"}
 
-		# Disable the abort logic if lvm logs too much, which easily happens
-		# when utilizing the lvm shell.
-		local_env["LVM_LOG_FILE_MAX_LINES"] = "0"
+		# If any env variables contain LVM we will propagate them too
+		for k, v in os.environ.items():
+			if "LVM" in k:
+				local_env[k] = v
+
+		self.parent_stdin_fd, child_stdin_fd = pty.openpty()
+		self.parent_stdout_fd, child_stdout_fd = pty.openpty()
+		self.parent_stderr_fd, child_stderr_fd = pty.openpty()
+		self.parent_stdin = os.fdopen(self.parent_stdin_fd, "w")
+		self.parent_stdout = os.fdopen(self.parent_stdout_fd, "r")
+		self.parent_stderr = os.fdopen(self.parent_stderr_fd, "r")
 
 		# run the lvm shell
 		self.lvm_shell = subprocess.Popen(
-			[LVM_CMD + " 32>%s" % tmp_file],
-			stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=local_env,
-			stderr=subprocess.PIPE, close_fds=True, shell=True)
+			[cfg.LVM_CMD],
+			stdin=child_stdin_fd,
+			stdout=child_stdout_fd, env=local_env,
+			stderr=child_stderr_fd, close_fds=True,
+			pass_fds=(lvm_fd,), shell=False)
 
 		try:
-			make_non_block(self.lvm_shell.stdout)
-			make_non_block(self.lvm_shell.stderr)
+			make_non_block(self.parent_stdout_fd)
+			make_non_block(self.parent_stderr_fd)
+
+			# Close our copies of the child FDs there were created with the fork, we don't need them open.
+			os.close(lvm_fd)
+			os.close(child_stdin_fd)
+			os.close(child_stdout_fd)
+			os.close(child_stderr_fd)
 
 			# wait for the first prompt
-			errors = self._read_until_prompt(no_output=True)[2]
+			log_debug("waiting for first prompt...")
+			errors = self._read_response(no_output=True)[2]
 			if errors and len(errors):
-				raise RuntimeError(errors)
+				raise LvmBug(errors)
+			log_debug("lvm prompt read!!!")
 		except:
 			raise
 		finally:
-			# These will get deleted when the FD count goes to zero so we
+			# These will get deleted when the FD count goes to zero, so we
 			# can be sure to clean up correctly no matter how we finish
 			os.unlink(tmp_file)
 			os.rmdir(tmp_dir)
 
-	def get_error_msg(self):
-		# We got an error, lets go fetch the error message
+	def get_last_log(self):
 		self._write_cmd('lastlog\n')
-
-		# read everything from the STDOUT to the next prompt
-		stdout, report_json, stderr = self._read_until_prompt()
-		if 'log' in report_json:
-			error_msg = ""
-			# Walk the entire log array and build an error string
-			for log_entry in report_json['log']:
-				if log_entry['log_type'] == "error":
-					if error_msg:
-						error_msg += ', ' + log_entry['log_message']
-					else:
-						error_msg = log_entry['log_message']
-
-			return error_msg
-
-		return 'No error reason provided! (missing "log" section)'
+		report_json = self._read_response()[1]
+		return get_error_msg(report_json)
 
 	def call_lvm(self, argv, debug=False):
 		rc = 1
@@ -204,21 +212,29 @@ class LVMShellProxy(object):
 		self._write_cmd(cmd)
 
 		# read everything from the STDOUT to the next prompt
-		stdout, report_json, stderr = self._read_until_prompt()
+		stdout, report_json, stderr = self._read_response()
 
 		# Parse the report to see what happened
 		if 'log' in report_json:
 			ret_code = int(report_json['log'][-1:][0]['log_ret_code'])
 			# If we have an exported vg we get a log_ret_code == 5 when
 			# we do a 'fullreport'
+			# Note: 0 == error
 			if (ret_code == 1) or (ret_code == 5 and argv[0] == 'fullreport'):
 				rc = 0
 			else:
-				error_msg = self.get_error_msg()
+				# Depending on where lvm fails the command, it may not have anything
+				# to report for "lastlog", so we need to check for a message in the
+				# report json too.
+				error_msg = self.get_last_log()
+				if error_msg is None:
+					error_msg = get_error_msg(report_json)
+					if error_msg is None:
+						error_msg = 'No error reason provided! (missing "log" section)'
 
 		if debug or rc != 0:
-			log_error(('CMD: %s' % cmd))
-			log_error(("EC = %d" % rc))
+			log_error(("CMD= %s" % cmd))
+			log_error(("EC= %d" % rc))
 			log_error(("ERROR_MSG=\n %s\n" % error_msg))
 
 		return rc, report_json, error_msg
@@ -226,35 +242,58 @@ class LVMShellProxy(object):
 	def exit_shell(self):
 		try:
 			self._write_cmd('exit\n')
-		except Exception as e:
-			log_error(str(e))
+			self.lvm_shell.wait(1)
+			self.lvm_shell = None
+		except Exception as _e:
+			log_error(str(_e))
 
 	def __del__(self):
-		try:
-			self.lvm_shell.terminate()
-		except:
-			pass
+		# Note: When we are shutting down the daemon and the main process has already exited
+		# and this gets called we have a limited set of things we can do, like we cannot call
+		# log_error as _common_log is None!!!
+		if self.lvm_shell is not None:
+			try:
+				self.lvm_shell.wait(1)
+			except subprocess.TimeoutExpired:
+				print("lvm shell child process did not exit as instructed, sending SIGTERM")
+				cfg.ignore_sigterm = True
+				self.lvm_shell.terminate()
+				child_exit_code = self.lvm_shell.wait(1)
+				print("lvm shell process exited with %d" % child_exit_code)
 
 
 if __name__ == "__main__":
-	shell = LVMShellProxy()
-	in_line = "start"
+	print("USING LVM BINARY: %s " % cfg.LVM_CMD)
+
 	try:
-		while in_line:
-			in_line = input("lvm> ")
-			if in_line:
-				start = time.time()
-				ret, out, err = shell.call_lvm(in_line.split())
-				end = time.time()
+		if len(sys.argv) > 1 and sys.argv[1] == "bisect":
+			shell = LVMShellProxy()
+			shell.exit_shell()
+		else:
+			shell = LVMShellProxy()
+			in_line = "start"
+			try:
+				while in_line:
+					in_line = input("lvm> ")
+					if in_line:
+						if in_line == "exit":
+							shell.exit_shell()
+							sys.exit(0)
+						start = time.time()
+						ret, out, err = shell.call_lvm(in_line.split())
+						end = time.time()
 
-				print(("RC: %d" % ret))
-				print(("OUT:\n%s" % out))
-				print(("ERR:\n%s" % err))
+						print(("RC: %d" % ret))
+						print(("OUT:\n%s" % out))
+						print(("ERR:\n%s" % err))
 
-				print("Command     = %f seconds" % (end - start))
-	except KeyboardInterrupt:
-		pass
-	except EOFError:
-		pass
-	except Exception:
-		traceback.print_exc(file=sys.stdout)
+						print("Command     = %f seconds" % (end - start))
+			except KeyboardInterrupt:
+				pass
+			except EOFError:
+				pass
+	except Exception as e:
+		log_error("main process exiting on exception!\n%s" % extract_stack_trace(e))
+		sys.exit(1)
+
+	sys.exit(0)

daemons/lvmdbusd/lvmdb.py.in

@@ -13,14 +13,13 @@ from collections import OrderedDict
 
 import pprint as prettyprint
 import os
-import sys
 
 from lvmdbusd import cmdhandler
-from lvmdbusd.utils import log_debug, log_error
+from lvmdbusd.utils import log_debug, log_error, lvm_column_key, LvmBug
 
 
 class DataStore(object):
-	def __init__(self, usejson=True, vdo_support=False):
+	def __init__(self, vdo_support=False):
 		self.pvs = {}
 		self.vgs = {}
 		self.lvs = {}
@@ -35,41 +34,9 @@ class DataStore(object):
 		self.lvs_in_vgs = {}
 		self.pvs_in_vgs = {}
 
-		# self.refresh()
 		self.num_refreshes = 0
-
-		if usejson:
-			self.json = cmdhandler.supports_json()
-		else:
-			self.json = usejson
-
 		self.vdo_support = vdo_support
 
-	@staticmethod
-	def _insert_record(table, key, record, allowed_multiple):
-		if key in table:
-			existing = table[key]
-
-			for rec_k, rec_v in record.items():
-				if rec_k in allowed_multiple:
-					# This column name allows us to store multiple value for
-					# each type
-					if not isinstance(existing[rec_k], list):
-						existing_value = existing[rec_k]
-						existing[rec_k] = [existing_value, rec_v]
-					else:
-						existing[rec_k].append(rec_v)
-				else:
-					# If something is not expected to have changing values
-					# lets ensure that
-					if existing[rec_k] != rec_v:
-						raise RuntimeError(
-							"existing[%s]=%s != %s" %
-							(rec_k, str(existing[rec_k]),
-							str(rec_v)))
-		else:
-			table[key] = record
-
 	@staticmethod
 	def _pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup):
 		for p in c_pvs.values():
@@ -84,22 +51,6 @@ class DataStore(object):
 			# Lookup for translating between /dev/<name> and pv uuid
 			c_lookup[p['pv_name']] = p['pv_uuid']
 
-	@staticmethod
-	def _parse_pvs(_pvs):
-		pvs = sorted(_pvs, key=lambda pk: pk['pv_name'])
-
-		c_pvs = OrderedDict()
-		c_lookup = {}
-		c_pvs_in_vgs = {}
-
-		for p in pvs:
-			DataStore._insert_record(
-				c_pvs, p['pv_uuid'], p,
-				['pvseg_start', 'pvseg_size', 'segtype'])
-
-		DataStore._pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup)
-		return c_pvs, c_lookup, c_pvs_in_vgs
-
 	@staticmethod
 	def _parse_pvs_json(_all):
 
@@ -107,7 +58,7 @@ class DataStore(object):
 		c_lookup = {}
 		c_pvs_in_vgs = {}
 
-		# Each item item in the report is a collection of information pertaining
+		# Each item in the report is a collection of information pertaining
 		# to the vg
 		for r in _all['report']:
 			tmp_pv = []
@@ -141,28 +92,6 @@ class DataStore(object):
 
 		return c_pvs, c_lookup, c_pvs_in_vgs
 
-	@staticmethod
-	def _parse_vgs(_vgs):
-		vgs = sorted(_vgs, key=lambda vk: vk['vg_uuid'])
-
-		c_vgs = OrderedDict()
-		c_lookup = {}
-
-		for i in vgs:
-			vg_name = i['vg_name']
-
-			# Lvm allows duplicate vg names.  When this occurs, each subsequent
-			# matching VG name will be called vg_name:vg_uuid.  Note: ':' is an
-			# invalid character for lvm VG names
-			if vg_name in c_lookup:
-				vg_name = "%s:%s" % (vg_name, i['vg_uuid'])
-				i['vg_name'] = vg_name
-
-			c_lookup[vg_name] = i['vg_uuid']
-			DataStore._insert_record(c_vgs, i['vg_uuid'], i, [])
-
-		return c_vgs, c_lookup
-
 	@staticmethod
 	def _parse_vgs_json(_all):
 
@@ -227,28 +156,12 @@ class DataStore(object):
 
 		return c_lvs, c_lvs_in_vgs, c_lvs_hidden, c_lv_full_lookup
 
-	@staticmethod
-	def _parse_lvs(_lvs):
-		lvs = sorted(_lvs, key=lambda vk: vk['lv_name'])
-
-		c_lvs = OrderedDict()
-		c_lv_full_lookup = OrderedDict()
-
-		for i in lvs:
-			full_name = "%s/%s" % (i['vg_name'], i['lv_name'])
-			c_lv_full_lookup[full_name] = i['lv_uuid']
-			DataStore._insert_record(
-				c_lvs, i['lv_uuid'], i,
-				['seg_pe_ranges', 'segtype'])
-
-		return DataStore._parse_lvs_common(c_lvs, c_lv_full_lookup)
-
 	def _parse_lvs_json(self, _all):
 
 		c_lvs = OrderedDict()
 		c_lv_full_lookup = {}
 
-		# Each item item in the report is a collection of information pertaining
+		# Each item in the report is a collection of information pertaining
 		# to the vg
 		for r in _all['report']:
 			# Get the lv data for this VG.
@@ -396,12 +309,12 @@ class DataStore(object):
 		:param log  Add debug log entry/exit messages
 		:return: None
 		"""
-		self.num_refreshes += 1
-		if log:
-			log_debug("lvmdb - refresh entry")
+		try:
+			self.num_refreshes += 1
+			if log:
+				log_debug("lvmdb - refresh entry")
 
-		# Grab everything first then parse it
-		if self.json:
+			# Grab everything first then parse it
 			# Do a single lvm retrieve for everything in json
 			a = cmdhandler.lvm_full_report_json()
 
@@ -409,29 +322,25 @@ class DataStore(object):
 			_vgs, _vgs_lookup = self._parse_vgs_json(a)
 			_lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs_json(a)
 
-		else:
-			_raw_pvs = cmdhandler.pv_retrieve_with_segs()
-			_raw_vgs = cmdhandler.vg_retrieve(None)
-			_raw_lvs = cmdhandler.lv_retrieve_with_segments()
+			# Set all
+			self.pvs = _pvs
+			self.pv_path_to_uuid = _pvs_lookup
+			self.vg_name_to_uuid = _vgs_lookup
+			self.lv_full_name_to_uuid = _lvs_lookup
 
-			_pvs, _pvs_lookup, _pvs_in_vgs = self._parse_pvs(_raw_pvs)
-			_vgs, _vgs_lookup = self._parse_vgs(_raw_vgs)
-			_lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs(_raw_lvs)
+			self.vgs = _vgs
+			self.lvs = _lvs
+			self.lvs_in_vgs = _lvs_in_vgs
+			self.pvs_in_vgs = _pvs_in_vgs
+			self.lvs_hidden = _lvs_hidden
 
-		# Set all
-		self.pvs = _pvs
-		self.pv_path_to_uuid = _pvs_lookup
-		self.vg_name_to_uuid = _vgs_lookup
-		self.lv_full_name_to_uuid = _lvs_lookup
-
-		self.vgs = _vgs
-		self.lvs = _lvs
-		self.lvs_in_vgs = _lvs_in_vgs
-		self.pvs_in_vgs = _pvs_in_vgs
-		self.lvs_hidden = _lvs_hidden
-
-		# Create lookup table for which LV and segments are on each PV
-		self.pv_lvs, self.lv_pvs = self._parse_pv_in_lvs()
+			# Create lookup table for which LV and segments are on each PV
+			self.pv_lvs, self.lv_pvs = self._parse_pv_in_lvs()
+		except KeyError as ke:
+			key = ke.args[0]
+			if lvm_column_key(key):
+				raise LvmBug("missing JSON key: '%s'" % key)
+			raise ke
 
 		if log:
 			log_debug("lvmdb - refresh exit")
@@ -451,10 +360,13 @@ class DataStore(object):
 			return rc
 
 	def pv_missing(self, pv_uuid):
+		# The uuid might not be a PV, default to false
 		if pv_uuid in self.pvs:
 			if self.pvs[pv_uuid]['pv_missing'] == '':
 				return False
-		return True
+			else:
+				return True
+		return False
 
 	def fetch_vgs(self, vg_name):
 		if not vg_name:
@@ -527,13 +439,7 @@ class DataStore(object):
 if __name__ == "__main__":
 	pp = prettyprint.PrettyPrinter(indent=4)
 
-	use_json = False
-
-	if len(sys.argv) != 1:
-		print(len(sys.argv))
-		use_json = True
-
-	ds = DataStore(use_json)
+	ds = DataStore()
 	ds.refresh()
 
 	print("PVS")

daemons/lvmdbusd/main.py

@@ -22,14 +22,13 @@ from . import lvmdb
 from gi.repository import GLib
 from .fetch import StateUpdate
 from .manager import Manager
-import traceback
 import queue
 from . import udevwatch
-from .utils import log_debug, log_error
+from .utils import log_debug, log_error, log_msg, DebugMessages
 import argparse
 import os
 import sys
-from .cmdhandler import LvmFlightRecorder, supports_vdo
+from .cmdhandler import LvmFlightRecorder, supports_vdo, supports_json
 from .request import RequestEntry
 
 
@@ -50,12 +49,15 @@ def process_request():
 			log_debug("Method complete: %s" % str(req.method))
 		except queue.Empty:
 			pass
-		except Exception:
-			st = traceback.format_exc()
+		except SystemExit:
+			break
+		except Exception as e:
+			st = utils.extract_stack_trace(e)
 			utils.log_error("process_request exception: \n%s" % st)
+	log_debug("process_request thread exiting!")
 
 
-def check_bb_size(value):
+def check_fr_size(value):
 	v = int(value)
 	if v < 0:
 		raise argparse.ArgumentTypeError(
@@ -65,7 +67,7 @@ def check_bb_size(value):
 
 def install_signal_handlers():
 	# Because of the glib main loop stuff the python signal handler code is
-	# apparently not usable and we need to use the glib calls instead
+	# apparently not usable, and we need to use the glib calls instead
 	signal_add = None
 
 	if hasattr(GLib, 'unix_signal_add'):
@@ -77,13 +79,13 @@ def install_signal_handlers():
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGHUP, utils.handler, signal.SIGHUP)
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGINT, utils.handler, signal.SIGINT)
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGUSR1, utils.handler, signal.SIGUSR1)
+		signal_add(GLib.PRIORITY_HIGH, signal.SIGUSR2, utils.handler, signal.SIGUSR2)
+		signal_add(GLib.PRIORITY_HIGH, signal.SIGTERM, utils.handler, signal.SIGTERM)
 	else:
 		log_error("GLib.unix_signal_[add|add_full] are NOT available!")
 
 
-def main():
-	start = time.time()
-	# Add simple command line handling
+def process_args():
 	parser = argparse.ArgumentParser()
 	parser.add_argument(
 		"--udev", action='store_true',
@@ -104,101 +106,142 @@ def main():
 		default=False,
 		dest='use_lvm_shell')
 	parser.add_argument(
-		"--blackboxsize",
-		help="Size of the black box flight recorder, 0 to disable",
+		"--frsize",
+		help="Size of the flight recorder (num. entries), 0 to disable (signal 12 to dump)",
 		default=10,
-		type=check_bb_size,
-		dest='bb_size')
+		type=check_fr_size,
+		dest='fr_size')
 
-	use_session = os.getenv('LVMDBUSD_USE_SESSION', False)
+	args = parser.parse_args()
 
-	# Ensure that we get consistent output for parsing stdout/stderr
+	if not args.use_json:
+		log_error("Daemon no longer supports lvm without JSON support, exiting!")
+		sys.exit(1)
+	else:
+		if not supports_json():
+			log_error("Un-supported version of LVM, daemon requires JSON output, exiting!")
+			sys.exit(1)
+
+	# Add udev watching
+	if args.use_udev:
+		# Make sure this msg ends up in the journal, so we know
+		log_msg('The --udev option is no longer supported,'
+				'the daemon always uses a combination of dbus notify from lvm tools and udev')
+
+	return args
+
+
+def running_under_systemd():
+	""""
+	Checks to see if we are running under systemd, by checking damon fd 0, 1
+	systemd sets stdin to /dev/null and 1 & 2 are a socket
+	"""
+	base = "/proc/self/fd"
+	stdout = os.readlink("%s/0" % base)
+	if stdout == "/dev/null":
+		stdout = os.readlink("%s/1" % base)
+		if "socket" in stdout:
+			return True
+	return False
+
+
+def main():
+	start = time.time()
+	use_session = os.getenv('LVM_DBUSD_USE_SESSION', False)
+	collect_lvm_debug = os.getenv('LVM_DBUSD_COLLECT_LVM_DEBUG', False)
+
+	# Ensure that we get consistent output for parsing stdout/stderr and that we
+	# are using the lvmdbusd profile.
 	os.environ["LC_ALL"] = "C"
+	os.environ["LVM_COMMAND_PROFILE"] = "lvmdbusd"
+
+	# Save off the debug data needed for lvm team to debug issues
+	# only used for 'fullreport' at this time.
+	cfg.lvmdebug = utils.LvmDebugData(collect_lvm_debug)
+
+	# Indicator if we are running under systemd
+	cfg.systemd = running_under_systemd()
+
+	# Add simple command line handling
+	cfg.args = process_args()
 
-	cfg.args = parser.parse_args()
 	cfg.create_request_entry = RequestEntry
 
 	# We create a flight recorder in cmdhandler too, but we replace it here
 	# as the user may be specifying a different size.  The default one in
 	# cmdhandler is for when we are running other code with a different main.
-	cfg.blackbox = LvmFlightRecorder(cfg.args.bb_size)
+	cfg.flightrecorder = LvmFlightRecorder(cfg.args.fr_size)
 
-	if cfg.args.use_lvm_shell and not cfg.args.use_json:
-		log_error("You cannot specify --lvmshell and --nojson")
-		sys.exit(1)
+	# Create a circular buffer for debug logs
+	cfg.debug = DebugMessages()
+
+	log_debug("Using lvm binary: %s" % cfg.LVM_CMD)
 
 	# We will dynamically add interfaces which support vdo if it
 	# exists.
 	cfg.vdo_support = supports_vdo()
 
-	if cfg.vdo_support and not cfg.args.use_json:
-		log_error("You cannot specify --nojson when lvm has VDO support")
-		sys.exit(1)
-
 	# List of threads that we start up
 	thread_list = []
 
 	install_signal_handlers()
 
-	dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
-	dbus.mainloop.glib.threads_init()
+	with utils.LockFile(cfg.LOCK_FILE):
+		dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+		dbus.mainloop.glib.threads_init()
 
-	cmdhandler.set_execution(cfg.args.use_lvm_shell)
+		cmdhandler.set_execution(cfg.args.use_lvm_shell)
 
-	if use_session:
-		cfg.bus = dbus.SessionBus()
-	else:
-		cfg.bus = dbus.SystemBus()
-	# The base name variable needs to exist for things to work.
-	# noinspection PyUnusedLocal
-	base_name = dbus.service.BusName(BUS_NAME, cfg.bus)
-	cfg.om = Lvm(BASE_OBJ_PATH)
-	cfg.om.register_object(Manager(MANAGER_OBJ_PATH))
+		if use_session:
+			cfg.bus = dbus.SessionBus()
+		else:
+			cfg.bus = dbus.SystemBus()
+		# The base name variable needs to exist for things to work.
+		# noinspection PyUnusedLocal
+		base_name = dbus.service.BusName(BUS_NAME, cfg.bus)
+		cfg.om = Lvm(BASE_OBJ_PATH)
+		cfg.om.register_object(Manager(MANAGER_OBJ_PATH))
 
-	cfg.db = lvmdb.DataStore(cfg.args.use_json, cfg.vdo_support)
+		cfg.db = lvmdb.DataStore(vdo_support=cfg.vdo_support)
 
-	# Using a thread to process requests, we cannot hang the dbus library
-	# thread that is handling the dbus interface
-	thread_list.append(
-		threading.Thread(target=process_request, name='process_request'))
+		# Using a thread to process requests, we cannot hang the dbus library
+		# thread that is handling the dbus interface
+		thread_list.append(
+			threading.Thread(target=process_request, name='process_request'))
 
-	# Have a single thread handling updating lvm and the dbus model so we
-	# don't have multiple threads doing this as the same time
-	updater = StateUpdate()
-	thread_list.append(updater.thread)
+		# Have a single thread handling updating lvm and the dbus model, so we
+		# don't have multiple threads doing this as the same time
+		updater = StateUpdate()
+		thread_list.append(updater.thread)
 
-	cfg.load = updater.load
+		cfg.load = updater.load
 
-	cfg.loop = GLib.MainLoop()
+		cfg.loop = GLib.MainLoop()
 
-	for thread in thread_list:
-		thread.damon = True
-		thread.start()
+		for thread in thread_list:
+			thread.damon = True
+			thread.start()
 
-	# Add udev watching
-	if cfg.args.use_udev:
-		log_debug('Utilizing udev to trigger updates')
+		# In all cases we are going to monitor for udev until we get an
+		# ExternalEvent.  In the case where we get an external event and the user
+		# didn't specify --udev we will stop monitoring udev
+		udevwatch.add()
 
-	# In all cases we are going to monitor for udev until we get an
-	# ExternalEvent.  In the case where we get an external event and the user
-	# didn't specify --udev we will stop monitoring udev
-	udevwatch.add()
+		end = time.time()
+		log_debug(
+			'Service ready! total time= %.4f, lvm time= %.4f count= %d' %
+			(end - start, cmdhandler.total_time, cmdhandler.total_count),
+			'bg_black', 'fg_light_green')
 
-	end = time.time()
-	log_debug(
-		'Service ready! total time= %.4f, lvm time= %.4f count= %d' %
-		(end - start, cmdhandler.total_time, cmdhandler.total_count),
-		'bg_black', 'fg_light_green')
+		try:
+			if cfg.run.value != 0:
+				cfg.loop.run()
+				udevwatch.remove()
 
-	try:
-		if cfg.run.value != 0:
-			cfg.loop.run()
-			udevwatch.remove()
-
-			for thread in thread_list:
-				thread.join()
-	except KeyboardInterrupt:
-		# If we are unable to register signal handler, we will end up here when
-		# the service gets a ^C or a kill -2 <parent pid>
-		utils.handler(signal.SIGINT)
+				for thread in thread_list:
+					thread.join()
+		except KeyboardInterrupt:
+			# If we are unable to register signal handler, we will end up here when
+			# the service gets a ^C or a kill -2 <parent pid>
+			utils.handler(signal.SIGINT)
 	return 0

daemons/lvmdbusd/manager.py

@@ -137,7 +137,8 @@ class Manager(AutomatedProperties):
 		"""
 		Dump the flight recorder to syslog
 		"""
-		cfg.blackbox.dump()
+		cfg.debug.dump()
+		cfg.flightrecorder.dump()
 
 	@staticmethod
 	def _lookup_by_lvm_id(key):
@@ -194,6 +195,7 @@ class Manager(AutomatedProperties):
 	def _external_event(command):
 		utils.log_debug("Processing _external_event= %s" % command,
 							'bg_black', 'fg_orange')
+		cfg.got_external_event = True
 		cfg.load()
 
 	@dbus.service.method(
@@ -201,14 +203,6 @@ class Manager(AutomatedProperties):
 		in_signature='s', out_signature='i')
 	def ExternalEvent(self, command):
 		utils.log_debug("ExternalEvent %s" % command)
-		# If a user didn't explicitly specify udev, we will turn it off now.
-		if not cfg.args.use_udev:
-			if udevwatch.remove():
-				utils.log_debug("ExternalEvent received, disabling "
-								"udev monitoring")
-				# We are dependent on external events now to stay current!
-				cfg.got_external_event = True
-
 		r = RequestEntry(
 			-1, Manager._external_event, (command,), None, None, False)
 		cfg.worker_q.put(r)

daemons/lvmdbusd/objectmanager.py

@@ -9,12 +9,11 @@
 
 import sys
 import threading
-import traceback
 import dbus
 import os
 import copy
 from . import cfg
-from .utils import log_debug, pv_obj_path_generate, log_error
+from .utils import log_debug, pv_obj_path_generate, log_error, extract_stack_trace
 from .automatedproperties import AutomatedProperties
 
 
@@ -40,8 +39,8 @@ class ObjectManager(AutomatedProperties):
 				for k, v in list(obj._objects.items()):
 					path, props = v[0].emit_data()
 					rc[path] = props
-			except Exception:
-				traceback.print_exc(file=sys.stdout)
+			except Exception as e:
+				log_error("_get_managed_objects exception, bailing: \n%s" % extract_stack_trace(e))
 				sys.exit(1)
 			return rc
 
@@ -53,15 +52,6 @@ class ObjectManager(AutomatedProperties):
 									(self, ), cb, cbe, False)
 		cfg.worker_q.put(r)
 
-	def locked(self):
-		"""
-		If some external code need to run across a number of different
-		calls into ObjectManager while blocking others they can use this method
-		to lock others out.
-		:return:
-		"""
-		return ObjectManagerLock(self.rlock)
-
 	@dbus.service.signal(
 		dbus_interface="org.freedesktop.DBus.ObjectManager",
 		signature='oa{sa{sv}}')
@@ -169,7 +159,7 @@ class ObjectManager(AutomatedProperties):
 			# print('Registering object path %s for %s' %
 			# (path, dbus_object.lvm_id))
 
-			# We want fast access to the object by a number of different ways
+			# We want fast access to the object by a number of different ways,
 			# so we use multiple hashs with different keys
 			self._lookup_add(dbus_object, path, dbus_object.lvm_id,
 				dbus_object.Uuid)
@@ -219,7 +209,7 @@ class ObjectManager(AutomatedProperties):
 
 	def get_object_by_lvm_id(self, lvm_id):
 		"""
-		Given an lvm identifier, return the object registered for it
+		Given a lvm identifier, return the object registered for it
 		:param lvm_id: The lvm identifier
 		"""
 		with self.rlock:
@@ -230,7 +220,7 @@ class ObjectManager(AutomatedProperties):
 
 	def get_object_path_by_lvm_id(self, lvm_id):
 		"""
-		Given an lvm identifier, return the object path for it
+		Given a lvm identifier, return the object path for it
 		:param lvm_id: The lvm identifier
 		:return: Object path or '/' if not found
 		"""
@@ -287,7 +277,7 @@ class ObjectManager(AutomatedProperties):
 		register it with the object manager for the specified uuid & lvm_id.
 		Note: If path create is not None, uuid and lvm_id cannot be equal
 		:param uuid: The uuid for the lvm object we are searching for
-		:param lvm_id: The lvm name (eg. pv device path, vg name, lv full name)
+		:param lvm_id: The lvm name (e.g. pv device path, vg name, lv full name)
 		:param path_create: If not None, create the path using this function if
 				we fail to find the object by uuid or lvm_id.
 		:returns None if lvm asset not found and path_create == None otherwise
@@ -305,18 +295,17 @@ class ObjectManager(AutomatedProperties):
 			if uuid == lvm_id:
 				path = self._id_lookup(lvm_id)
 			else:
-				# We have a uuid and a lvm_id we can do sanity checks to ensure
+				# We have an uuid and a lvm_id we can do sanity checks to ensure
 				# that they are consistent
 
-				# If a PV is missing it's device path is '[unknown]' or some
+				# If a PV is missing its device path is '[unknown]' or some
 				# other text derivation of unknown.  When we find that a PV is
-				# missing we will clear out the lvm_id as it's likely not unique
-				# and thus not useful and potentially harmful for lookups.
-				if path_create == pv_obj_path_generate and \
-						cfg.db.pv_missing(uuid):
+				# missing we will clear out the lvm_id as it's not unique
+				# and thus not useful and harmful for lookups.
+				if cfg.db.pv_missing(uuid):
 					lvm_id = None
 
-				# Lets check for the uuid first
+				# Let's check for the uuid first
 				path = self._id_lookup(uuid)
 				if path:
 					# Ensure table lookups are correct
@@ -337,29 +326,3 @@ class ObjectManager(AutomatedProperties):
 			#	(uuid, lvm_id, str(path_create), path))
 
 			return path
-
-
-class ObjectManagerLock(object):
-	"""
-	The sole purpose of this class is to allow other code the ability to
-	lock the object manager using a `with` statement, eg.
-
-	with cfg.om.locked():
-		# Do stuff with object manager
-
-	This will ensure that the lock is always released (assuming this is done
-	correctly)
-	"""
-
-	def __init__(self, recursive_lock):
-		self._lock = recursive_lock
-
-	def __enter__(self):
-		# Acquire lock
-		self._lock.acquire()
-
-	# noinspection PyUnusedLocal
-	def __exit__(self, e_type, e_value, e_traceback):
-		# Release lock
-		self._lock.release()
-		self._lock = None

daemons/lvmdbusd/pv.py

@@ -14,11 +14,11 @@ import dbus
 from .cfg import PV_INTERFACE
 from . import cmdhandler
 from .utils import vg_obj_path_generate, n, pv_obj_path_generate, \
-	lv_object_path_method, _handle_execute
+	lv_object_path_method, _handle_execute, lvm_column_key
 from .loader import common
 from .request import RequestEntry
 from .state import State
-from .utils import round_size
+from .utils import round_size, LvmBug
 
 
 # noinspection PyUnusedLocal
@@ -28,16 +28,23 @@ def pvs_state_retrieve(selection, cache_refresh=True):
 	if cache_refresh:
 		cfg.db.refresh()
 
-	for p in cfg.db.fetch_pvs(selection):
-		rc.append(
-			PvState(
-				p["pv_name"], p["pv_uuid"], p["pv_name"],
-				p["pv_fmt"], n(p["pv_size"]), n(p["pv_free"]),
-				n(p["pv_used"]), n(p["dev_size"]), n(p["pv_mda_size"]),
-				n(p["pv_mda_free"]), int(p["pv_ba_start"]),
-				n(p["pv_ba_size"]), n(p["pe_start"]),
-				int(p["pv_pe_count"]), int(p["pv_pe_alloc_count"]),
-				p["pv_attr"], p["pv_tags"], p["vg_name"], p["vg_uuid"]))
+	try:
+		for p in cfg.db.fetch_pvs(selection):
+			rc.append(
+				PvState(
+					p["pv_name"], p["pv_uuid"], p["pv_name"],
+					p["pv_fmt"], n(p["pv_size"]), n(p["pv_free"]),
+					n(p["pv_used"]), n(p["dev_size"]), n(p["pv_mda_size"]),
+					n(p["pv_mda_free"]), int(p["pv_ba_start"]),
+					n(p["pv_ba_size"]), n(p["pe_start"]),
+					int(p["pv_pe_count"]), int(p["pv_pe_alloc_count"]),
+					p["pv_attr"], p["pv_tags"], p["vg_name"], p["vg_uuid"]))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 

daemons/lvmdbusd/request.py

@@ -7,13 +7,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+import dbus
 import threading
 # noinspection PyUnresolvedReferences
 from gi.repository import GLib
 from .job import Job
 from . import cfg
-import traceback
-from .utils import log_error, mt_async_call
+from .utils import log_error, mt_async_call, extract_stack_trace
 
 
 class RequestEntry(object):
@@ -71,13 +71,23 @@ class RequestEntry(object):
 		try:
 			result = self.method(*self.arguments)
 			self.register_result(result)
+		except SystemExit as se:
+			self.register_error(-1, str(se), se)
+			raise se
+		except dbus.exceptions.DBusException as dbe:
+			# This is an expected error path when something goes awry that
+			# we handled
+			self.register_error(-1, str(dbe), dbe)
 		except Exception as e:
 			# Use the request entry to return the result as the client may
 			# have gotten a job by the time we hit an error
-			# Lets get the stacktrace and set that to the error message
-			st = traceback.format_exc()
-			cfg.blackbox.dump()
-			log_error("Exception returned to client: \n%s" % st)
+			# Lets set the exception text as the error message and log the
+			# exception in the journal for figuring out what went wrong.
+			cfg.debug.dump()
+			cfg.flightrecorder.dump()
+			tb = extract_stack_trace(e)
+			log_error("While processing %s: we encountered\n%s" % (str(self.method), tb))
+			log_error("Error returned to client: %s" % str(e))
 			self.register_error(-1, str(e), e)
 
 	def is_done(self):
@@ -131,7 +141,7 @@ class RequestEntry(object):
 
 						mt_async_call(self.cb_error, error_exception)
 			else:
-				# We have a job and it's complete, indicate that it's done.
+				# We have a job, and it's complete, indicate that it's done.
 				self._job.Complete = True
 				self._job = None
 

daemons/lvmdbusd/udevwatch.py

@@ -52,20 +52,30 @@ def filter_event(action, device):
 	# when appropriate.
 	refresh = False
 
+	# Ignore everything but change
+	if action != 'change':
+		return
+
 	if 'ID_FS_TYPE' in device:
 		fs_type_new = device['ID_FS_TYPE']
-
 		if 'LVM' in fs_type_new:
-			refresh = True
+			# If we get a lvm related udev event for a block device
+			# we don't know about, it's either a pvcreate which we
+			# would handle with the dbus notification or something
+			# copied a pv signature onto a block device, this is
+			# required to catch the latter.
+			if not cfg.om.get_object_by_lvm_id(device['DEVNAME']):
+				refresh = True
 		elif fs_type_new == '':
 			# Check to see if the device was one we knew about
 			if 'DEVNAME' in device:
-				found = cfg.om.get_object_by_lvm_id(device['DEVNAME'])
-				if found:
+				if cfg.om.get_object_by_lvm_id(device['DEVNAME']):
 					refresh = True
-
-	if 'DM_LV_NAME' in device:
-		refresh = True
+	else:
+		# This handles the wipefs -a path
+		if not refresh and 'DEVNAME' in device:
+			if cfg.om.get_object_by_lvm_id(device['DEVNAME']):
+				refresh = True
 
 	if refresh:
 		udev_add()

daemons/lvmdbusd/utils.py

@@ -10,11 +10,15 @@
 import xml.etree.ElementTree as Et
 import sys
 import inspect
+import collections
 import ctypes
+import errno
+import fcntl
 import os
+import stat
 import string
 import datetime
-from fcntl import fcntl, F_GETFL, F_SETFL
+import tempfile
 
 import dbus
 from lvmdbusd import cfg
@@ -86,7 +90,7 @@ def init_class_from_arguments(
 			nt = k
 
 			# If the current attribute has a value, but the incoming does
-			# not, don't overwrite it.  Otherwise the default values on the
+			# not, don't overwrite it.  Otherwise, the default values on the
 			# property decorator don't work as expected.
 			cur = getattr(obj_instance, nt, v)
 
@@ -106,7 +110,7 @@ def init_class_from_arguments(
 
 def get_properties(f):
 	"""
-	Walks through an object instance or it's parent class(es) and determines
+	Walks through an object instance, or it's parent class(es) and determines
 	which attributes are properties and if they were created to be used for
 	dbus.
 	:param f:   Object to inspect
@@ -190,7 +194,7 @@ def add_properties(xml, interface, props):
 				interface_element = c
 				break
 
-		# Interface is not present, lets create it so we have something to
+		# Interface is not present, lets create it, so we have something to
 		# attach the properties too
 		if interface_element is None:
 			interface_element = Et.Element("interface", name=interface)
@@ -280,17 +284,47 @@ def parse_tags(tags):
 	return dbus.Array([], signature='s')
 
 
-def _common_log(msg, *attributes):
-	cfg.stdout_lock.acquire()
+class DebugMessages(object):
+
+	def __init__(self, size=5000):
+		self.queue = collections.deque(maxlen=size)
+		self.lock = threading.RLock()
+
+	def add(self, message):
+		with self.lock:
+			self.queue.append(message)
+
+	def dump(self):
+		if cfg.args and not cfg.args.debug:
+			with self.lock:
+				if len(self.queue):
+					log_error("LVM dbus debug messages START last (%d max) messages" % self.queue.maxlen)
+					for m in self.queue:
+						print(m)
+					log_error("LVM dbus debug messages END")
+					self.queue.clear()
+
+
+def _format_log_entry(msg):
 	tid = ctypes.CDLL('libc.so.6').syscall(186)
 
-	if STDOUT_TTY:
+	if not cfg.systemd and STDOUT_TTY:
 		msg = "%s: %d:%d - %s" % \
 			(datetime.datetime.now().strftime("%b %d %H:%M:%S.%f"),
 			os.getpid(), tid, msg)
 
 	else:
-		msg = "%d:%d - %s" % (os.getpid(), tid, msg)
+		if cfg.systemd:
+			# Systemd already puts the daemon pid in the log, we'll just add the tid
+			msg = "[%d]: %s" % (tid, msg)
+		else:
+			msg = "[%d:%d]: %s" % (os.getpid(), tid, msg)
+	return msg
+
+
+def _common_log(msg, *attributes):
+	cfg.stdout_lock.acquire()
+	msg = _format_log_entry(msg)
 
 	if STDOUT_TTY and attributes:
 		print(color(msg, *attributes))
@@ -307,12 +341,19 @@ def _common_log(msg, *attributes):
 def log_debug(msg, *attributes):
 	if cfg.args and cfg.args.debug:
 		_common_log(msg, *attributes)
+	else:
+		if cfg.debug:
+			cfg.debug.add(_format_log_entry(msg))
 
 
 def log_error(msg, *attributes):
 	_common_log(msg, *attributes)
 
 
+def log_msg(msg, *attributes):
+	_common_log(msg, *attributes)
+
+
 def dump_threads_stackframe():
 	ident_to_name = {}
 
@@ -340,15 +381,32 @@ def dump_threads_stackframe():
 # noinspection PyUnusedLocal
 def handler(signum):
 	try:
+		# signal 10
 		if signum == signal.SIGUSR1:
+			cfg.debug.dump()
 			dump_threads_stackframe()
+		# signal 12
+		elif signum == signal.SIGUSR2:
+			cfg.debug.dump()
+			cfg.flightrecorder.dump()
 		else:
+			# If we are getting a SIGTERM, and we sent one to the lvm shell we
+			# will ignore this and keep running.
+			if signum == signal.SIGTERM and cfg.ignore_sigterm:
+				# Clear the flag, so we will exit on SIGTERM if we didn't
+				# send it.
+				cfg.ignore_sigterm = False
+				return True
+
+			# If lvm shell is in use, tell it to exit
+			if cfg.SHELL_IN_USE is not None:
+				cfg.SHELL_IN_USE.exit_shell()
 			cfg.run.value = 0
-			log_debug('Exiting daemon with signal %d' % signum)
+			log_error('Exiting daemon with signal %d' % signum)
 			if cfg.loop is not None:
 				cfg.loop.quit()
-	except:
-		st = traceback.format_exc()
+	except BaseException as be:
+		st = extract_stack_trace(be)
 		log_error("signal handler: exception (logged, not reported!) \n %s" % st)
 
 	# It's important we report that we handled the exception for the exception
@@ -572,6 +630,23 @@ def validate_tag(interface, tag):
 			% (tag, _ALLOWABLE_TAG_CH))
 
 
+def add_config_option(cmdline, key, value):
+	if 'help' in cmdline:
+		return cmdline
+
+	if key in cmdline:
+		for i, arg in enumerate(cmdline):
+			if arg == key:
+				if len(cmdline) <= i + 1:
+					raise dbus.exceptions.DBusException("Missing value for --config option.")
+				cmdline[i + 1] += " %s" % value
+				break
+	else:
+		cmdline.extend([key, value])
+
+	return cmdline
+
+
 def add_no_notify(cmdline):
 	"""
 	Given a command line to execute we will see if `--config` is present, if it
@@ -583,27 +658,18 @@ def add_no_notify(cmdline):
 	:rtype: list
 	"""
 
-	# Only after we have seen an external event will be disable lvm from sending
+	# Only after we have seen an external event will we disable lvm from sending
 	# us one when we call lvm
+	rv = cmdline
 	if cfg.got_external_event:
-		if 'help' in cmdline:
-			return cmdline
+		rv = add_config_option(rv, "--config", "global/notify_dbus=0")
 
-		if '--config' in cmdline:
-			for i, arg in enumerate(cmdline):
-				if arg == '--config':
-					if len(cmdline) <= i+1:
-						raise dbus.exceptions.DBusException("Missing value for --config option.")
-					cmdline[i+1] += " global/notify_dbus=0"
-					break
-		else:
-			cmdline.extend(['--config', 'global/notify_dbus=0'])
-	return cmdline
+	return rv
 
 
 # The methods below which start with mt_* are used to execute the desired code
-# on the the main thread of execution to alleviate any issues the dbus-python
-# library with regards to multi-threaded access.  Essentially, we are trying to
+# on the main thread of execution to alleviate any issues the dbus-python
+# library with regard to multithreaded access.  Essentially, we are trying to
 # ensure all dbus library interaction is done from the same thread!
 
 
@@ -617,9 +683,8 @@ def _async_handler(call_back, parameters):
 			call_back(*parameters)
 		else:
 			call_back()
-	except:
-		st = traceback.format_exc()
-		log_error("mt_async_call: exception (logged, not reported!) \n %s" % st)
+	except BaseException as be:
+		log_error("mt_async_call: exception (logged, not reported!) \n %s" % extract_stack_trace(be))
 
 
 # Execute the function on the main thread with the provided parameters, do
@@ -669,9 +734,6 @@ class MThreadRunner(object):
 				self.rc = self.f()
 		except BaseException as be:
 			self.exception = be
-			st = traceback.format_exc()
-			log_error("MThreadRunner: exception \n %s" % st)
-			log_error("Exception will be raised in calling thread!")
 
 
 def _remove_objects(dbus_objects_rm):
@@ -686,8 +748,8 @@ def mt_remove_dbus_objects(objs):
 
 # Make stream non-blocking
 def make_non_block(stream):
-	flags = fcntl(stream, F_GETFL)
-	fcntl(stream, F_SETFL, flags | os.O_NONBLOCK)
+	flags = fcntl.fcntl(stream, fcntl.F_GETFL)
+	fcntl.fcntl(stream, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 
 
 def read_decoded(stream):
@@ -695,3 +757,127 @@ def read_decoded(stream):
 	if tmp:
 		return tmp.decode("utf-8")
 	return ''
+
+
+class LockFile(object):
+	"""
+	Simple lock file class
+	Based on Pg.1144 "The Linux Programming Interface" by Michael Kerrisk
+	"""
+	def __init__(self, lock_file):
+		self.fd = 0
+		self.lock_file = lock_file
+
+	def __enter__(self):
+		try:
+			self.fd = os.open(self.lock_file, os.O_CREAT | os.O_RDWR, stat.S_IRUSR | stat.S_IWUSR)
+
+			# Get and set the close on exec and lock the file
+			flags = fcntl.fcntl(self.fd, fcntl.F_GETFD)
+			flags |= fcntl.FD_CLOEXEC
+			fcntl.fcntl(self.fd, fcntl.F_SETFL, flags)
+			fcntl.lockf(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+		except OSError as e:
+			if e.errno == errno.EAGAIN:
+				log_error("Daemon already running, exiting!")
+			else:
+				log_error("Error during creation of lock file(%s): errno(%d), exiting!" % (self.lock_file, e.errno))
+			sys.exit(114)
+
+	def __exit__(self, _type, _value, _traceback):
+		os.close(self.fd)
+
+
+def extract_stack_trace(exception):
+	return ''.join(traceback.format_exception(None, exception, exception.__traceback__))
+
+
+def lvm_column_key(key):
+	# Check LV
+	if key.startswith("lv_") or key.startswith("vg_") or key.startswith("pool_") or \
+			key.endswith("_percent") or key.startswith("move_") or key.startswith("vdo_") or \
+			key in ["origin_uuid", "segtype", "origin", "data_lv", "metadata_lv"]:
+		return True
+	# Check VG
+	if key.startswith("vg_") or key.startswith("lv_") or key.startswith("pv_") or \
+			key in ["max_lv", "max_pv", "snap_count"]:
+		return True
+	# Check PV
+	if key.startswith("pv") or key.startswith("vg") or (key in ['dev_size', 'pe_start']):
+		return True
+	return False
+
+class LvmBug(RuntimeError):
+	"""
+	Things that are clearly a bug with lvm itself.
+	"""
+	def __init__(self, msg):
+		super().__init__(msg)
+
+	def __str__(self):
+		return "lvm bug encountered: %s" % ' '.join(self.args)
+
+
+class LvmDebugData:
+	def __init__(self, do_collection):
+		self.fd = -1
+		self.fn = None
+		self.collect = do_collection
+		if self.collect:
+			log_msg("Collecting lvm debug data!")
+
+	def _remove_file(self):
+		if self.fn is not None:
+			os.unlink(self.fn)
+			self.fn = None
+
+	def _close_fd(self):
+		if self.fd != -1:
+			os.close(self.fd)
+			self.fd = -1
+
+	def setup(self):
+		# Create a secure filename
+		if self.collect:
+			self.fd, self.fn = tempfile.mkstemp(suffix=".log", prefix="lvmdbusd.lvm.debug.")
+			return self.fn
+		return None
+
+	def lvm_complete(self):
+		# Remove the file ASAP, so we decrease our odds of leaving it
+		# around if the daemon gets killed by a signal -9
+		self._remove_file()
+
+	def dump(self):
+		# Read the file and log it to log_err
+		if self.fd != -1:
+			# How big could the verbose debug get?
+			debug = os.read(self.fd, 1024*1024*5)
+			debug_txt = debug.decode("utf-8")
+			for line in debug_txt.split("\n"):
+				log_error("lvm debug >>> %s" % line)
+			self._close_fd()
+		# In case lvm_complete doesn't get called.
+		self._remove_file()
+
+	def complete(self):
+		self._close_fd()
+		# In case lvm_complete doesn't get called.
+		self._remove_file()
+
+
+def get_error_msg(report_json):
+	# Get the error message from the returned JSON
+	if 'log' in report_json:
+		error_msg = ""
+		# Walk the entire log array and build an error string
+		for log_entry in report_json['log']:
+			if log_entry['log_type'] == "error":
+				if error_msg:
+					error_msg += ', ' + log_entry['log_message']
+				else:
+					error_msg = log_entry['log_message']
+
+		return error_msg
+
+	return None

daemons/lvmdbusd/vg.py

@@ -20,7 +20,7 @@ from .request import RequestEntry
 from .loader import common
 from .state import State
 from . import background
-from .utils import round_size, mt_remove_dbus_objects
+from .utils import round_size, mt_remove_dbus_objects, LvmBug, lvm_column_key
 from .job import JobState
 
 
@@ -31,17 +31,24 @@ def vgs_state_retrieve(selection, cache_refresh=True):
 	if cache_refresh:
 		cfg.db.refresh()
 
-	for v in cfg.db.fetch_vgs(selection):
-		rc.append(
-			VgState(
-				v['vg_uuid'], v['vg_name'], v['vg_fmt'], n(v['vg_size']),
-				n(v['vg_free']), v['vg_sysid'], n(v['vg_extent_size']),
-				n(v['vg_extent_count']), n(v['vg_free_count']),
-				v['vg_profile'], n(v['max_lv']), n(v['max_pv']),
-				n(v['pv_count']), n(v['lv_count']), n(v['snap_count']),
-				n(v['vg_seqno']), n(v['vg_mda_count']),
-				n(v['vg_mda_free']), n(v['vg_mda_size']),
-				n(v['vg_mda_used_count']), v['vg_attr'], v['vg_tags']))
+	try:
+		for v in cfg.db.fetch_vgs(selection):
+			rc.append(
+				VgState(
+					v['vg_uuid'], v['vg_name'], v['vg_fmt'], n(v['vg_size']),
+					n(v['vg_free']), v['vg_sysid'], n(v['vg_extent_size']),
+					n(v['vg_extent_count']), n(v['vg_free_count']),
+					v['vg_profile'], n(v['max_lv']), n(v['max_pv']),
+					n(v['pv_count']), n(v['lv_count']), n(v['snap_count']),
+					n(v['vg_seqno']), n(v['vg_mda_count']),
+					n(v['vg_mda_free']), n(v['vg_mda_size']),
+					n(v['vg_mda_used_count']), v['vg_attr'], v['vg_tags']))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 

daemons/lvmlockd/Makefile.in

@@ -15,45 +15,45 @@ srcdir = @srcdir@
 top_srcdir = @top_srcdir@
 top_builddir = @top_builddir@
 
-USE_SD_NOTIFY=yes
-
 SOURCES = lvmlockd-core.c
+SOURCES2 = lvmlockctl.c
+
+TARGETS = lvmlockd lvmlockctl
+
+include $(top_builddir)/make.tmpl
 
 ifeq ("@BUILD_LOCKDSANLOCK@", "yes")
   SOURCES += lvmlockd-sanlock.c
-  LOCK_LIBS += -lsanlock_client
+  CFLAGS += $(LIBSANLOCKCLIENT_CFLAGS)
+  LOCK_LIBS += $(LIBSANLOCKCLIENT_LIBS)
 endif
 
 ifeq ("@BUILD_LOCKDDLM@", "yes")
   SOURCES += lvmlockd-dlm.c
-  LOCK_LIBS += -ldlm_lt
-  LOCK_LIBS += -ldlmcontrol
+  CFLAGS += $(LIBDLM) $(LIBDLMCONTROL_CFLAGS)
+#  LOCK_LIBS += $(LIBDLM_LIBS) $(LIBDLMCONTROL_LIBS)
+  LOCK_LIBS += -ldlm_lt $(LIBDLMCONTROL_LIBS)
 endif
 
 ifeq ("@BUILD_LOCKDIDM@", "yes")
   SOURCES += lvmlockd-idm.c
-  LOCK_LIBS += -lseagate_ilm -lblkid
+#  LOCK_LIBS += $(LIBSEAGATEILM_LIBS) $(BLKID_LIBS)
+  LOCK_LIBS += -lseagate_ilm $(BLKID_LIBS)
 endif
 
-SOURCES2 = lvmlockctl.c
-
-TARGETS = lvmlockd lvmlockctl
-
 CFLOW_SOURCES = $(addprefix $(srcdir)/, $(SOURCES))
 CFLOW_TARGET = lvmlockd
 
 .PHONY: install_lvmlockd install_lvmlockctl
 
-include $(top_builddir)/make.tmpl
-
 CFLAGS += $(EXTRA_EXEC_CFLAGS)
 INCLUDES += -I$(top_srcdir)/libdaemon/server
 LDFLAGS += -L$(top_builddir)/libdaemon/server $(EXTRA_EXEC_LDFLAGS) $(ELDFLAGS)
 LIBS += $(DAEMON_LIBS) $(PTHREAD_LIBS)
 
-ifeq ($(USE_SD_NOTIFY),yes)
-	CFLAGS += $(shell pkg-config --cflags libsystemd) -DUSE_SD_NOTIFY
-	LIBS += $(shell pkg-config --libs libsystemd)
+ifneq (,$(firstword $(LIBSYSTEMD_LIBS)))
+	CFLAGS += $(LIBSYSTEMD_CFLAGS) -DUSE_SD_NOTIFY
+	LIBS += $(LIBSYSTEMD_LIBS)
 endif
 
 lvmlockd: $(OBJECTS) $(top_builddir)/libdaemon/server/libdaemonserver.a $(INTERNAL_LIBS)

daemons/lvmlockd/lvmlockd-core.c

@@ -1279,7 +1279,7 @@ static int res_lock(struct lockspace *ls, struct resource *r, struct action *act
 	 * be invalidated.  When we need to invalidate the lvmetad
 	 * cache, but don't have a usable r_version from the lvb,
 	 * send lvmetad new_version 0 which causes it to invalidate
-	 * the VG metdata without comparing against the currently
+	 * the VG metadata without comparing against the currently
 	 * cached VG seqno.
 	 */
 
@@ -3016,9 +3016,7 @@ static int add_lockspace_thread(const char *ls_name,
 				!alloc_and_copy_pvs_path(&ls2->pvs, &ls->pvs)) {
 			log_debug("add_lockspace_thread %s fails to allocate pvs", ls->name);
 			rv = -ENOMEM;
-		}
-
-		if (ls2->thread_stop) {
+		} else if (ls2->thread_stop) {
 			log_debug("add_lockspace_thread %s exists and stopping", ls->name);
 			rv = -EAGAIN;
 		} else if (!ls2->create_fail && !ls2->create_done) {
@@ -4779,7 +4777,7 @@ static void client_recv_action(struct client *cl)
 	const char *path;
 	const char *str;
 	struct pvs pvs;
-	char buf[17];	/* "path[%d]\0", %d outputs signed integer so max to 10 bytes */
+	char buf[18];	/* "path[%d]\0", %d outputs signed integer so max to 10 bytes */
 	int64_t val;
 	uint32_t opts = 0;
 	int result = 0;
@@ -5956,7 +5954,18 @@ static void adopt_locks(void)
 	}
 
 
-	/* FIXME: purge any remaining orphan locks in each rejoined ls? */
+	/* Try to purge the orphan locks when lock manager is dlm */
+	if (lm_support_dlm() && lm_is_running_dlm()) {
+		list_for_each_entry(ls, &ls_found, list) {
+			pthread_mutex_lock(&lockspaces_mutex);
+			ls1 = find_lockspace_name(ls->name);
+			if (ls1) {
+				log_debug("ls: %s purge locks", ls->name);
+				lm_purge_locks_dlm(ls1);
+			}
+			pthread_mutex_unlock(&lockspaces_mutex);
+		}
+	}
 
 	if (count_start_fail || count_adopt_fail)
 		goto fail;

daemons/lvmlockd/lvmlockd-dlm.c

@@ -220,6 +220,86 @@ int lm_prepare_lockspace_dlm(struct lockspace *ls)
 	return 0;
 }
 
+#define DLM_COMMS_PATH "/sys/kernel/config/dlm/cluster/comms"
+#define LOCK_LINE_MAX 1024
+static int get_local_nodeid(void)
+{
+	struct dirent *de;
+	DIR *ls_dir;
+	char ls_comms_path[PATH_MAX];
+	FILE *file = NULL;
+	char line[LOCK_LINE_MAX];
+	int rv = -1, val;
+
+	memset(ls_comms_path, 0, sizeof(ls_comms_path));
+	snprintf(ls_comms_path, PATH_MAX, "%s",DLM_COMMS_PATH);
+
+	if (!(ls_dir = opendir(ls_comms_path)))
+		return -ECONNREFUSED;
+
+	while ((de = readdir(ls_dir))) {
+		if (de->d_name[0] == '.')
+			continue;
+		memset(ls_comms_path, 0, sizeof(ls_comms_path));
+		snprintf(ls_comms_path, PATH_MAX, "%s/%s/local",
+		     DLM_COMMS_PATH, de->d_name);
+		file = fopen(ls_comms_path, "r");
+		if (!file)
+			continue;
+		if (fgets(line, LOCK_LINE_MAX, file)) {
+			fclose(file);
+			rv = sscanf(line, "%d", &val);
+			if ((rv == 1) && (val == 1 )) {
+				memset(ls_comms_path, 0, sizeof(ls_comms_path));
+				snprintf(ls_comms_path, PATH_MAX, "%s/%s/nodeid",
+				    DLM_COMMS_PATH, de->d_name);
+				file = fopen(ls_comms_path, "r");
+				if (!file)
+					continue;
+				if (fgets(line, LOCK_LINE_MAX, file)) {
+					rv = sscanf(line, "%d", &val);
+					if (rv == 1) {
+						fclose(file);
+						closedir(ls_dir);
+						return val;
+					}
+				}
+			}
+		}
+		fclose(file);
+	}
+
+	if (closedir(ls_dir))
+		log_error("get_local_nodeid closedir error");
+	return rv;
+}
+
+int lm_purge_locks_dlm(struct lockspace *ls)
+{
+	struct lm_dlm *lmd = (struct lm_dlm *)ls->lm_data;
+	int nodeid;
+	int rv = -1;
+
+	if (!lmd || !lmd->dh) {
+		log_error("purge_locks_dlm %s no dlm_handle_t error", ls->name);
+		goto fail;
+	}
+
+	nodeid = get_local_nodeid();
+	if (nodeid < 0) {
+		log_error("failed to get local nodeid");
+		goto fail;
+	}
+	if (dlm_ls_purge(lmd->dh, nodeid, 0)) {
+		log_error("purge_locks_dlm %s error", ls->name);
+		goto fail;
+	}
+
+	rv = 0;
+fail:
+	return rv;
+}
+
 int lm_add_lockspace_dlm(struct lockspace *ls, int adopt)
 {
 	struct lm_dlm *lmd = (struct lm_dlm *)ls->lm_data;

daemons/lvmlockd/lvmlockd-internal.h

@@ -392,6 +392,7 @@ static inline const char *mode_str(int x)
 int lm_init_vg_dlm(char *ls_name, char *vg_name, uint32_t flags, char *vg_args);
 int lm_prepare_lockspace_dlm(struct lockspace *ls);
 int lm_add_lockspace_dlm(struct lockspace *ls, int adopt);
+int lm_purge_locks_dlm(struct lockspace *ls);
 int lm_rem_lockspace_dlm(struct lockspace *ls, int free_vg);
 int lm_lock_dlm(struct lockspace *ls, struct resource *r, int ld_mode,
 		struct val_blk *vb_out, int adopt);
@@ -429,6 +430,11 @@ static inline int lm_add_lockspace_dlm(struct lockspace *ls, int adopt)
 	return -1;
 }
 
+static inline int lm_purge_locks_dlm(struct lockspace *ls)
+{
+	return -1;
+}
+
 static inline int lm_rem_lockspace_dlm(struct lockspace *ls, int free_vg)
 {
 	return -1;

daemons/lvmlockd/lvmlockd-sanlock.c

@@ -684,10 +684,10 @@ int lm_init_vg_sanlock(char *ls_name, char *vg_name, uint32_t flags, char *vg_ar
 			break;
 		}
 
-		if (rv) {
+		if (rv < 0) {
 			log_error("clear lv resource area %llu error %d",
 				  (unsigned long long)offset, rv);
-			break;
+			return rv;
 		}
 		offset += align_size;
 	}
@@ -1576,10 +1576,8 @@ int lm_rem_lockspace_sanlock(struct lockspace *ls, int free_vg)
 		goto out;
 
 	rv = sanlock_rem_lockspace(&lms->ss, 0);
-	if (rv < 0) {
+	if (rv < 0)
 		log_error("S %s rem_lockspace_san error %d", ls->name, rv);
-		return rv;
-	}
 
 	if (free_vg) {
 		/*

daemons/lvmpolld/lvmpolld-core.c

@@ -52,7 +52,7 @@ static pthread_key_t key;
 
 static const char *_strerror_r(int errnum, struct lvmpolld_thread_data *data)
 {
-#ifdef _GNU_SOURCE
+#if defined(_GNU_SOURCE) && defined(STRERROR_R_CHAR_P)
 	return strerror_r(errnum, data->buf, sizeof(data->buf)); /* never returns NULL */
 #elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600)
 	return strerror_r(errnum, data->buf, sizeof(data->buf)) ? "" : data->buf;

device_mapper/Makefile

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+# Copyright (C) 2018 - 2022 Red Hat, Inc. All rights reserved.
 #
 # This file is part of the device-mapper userspace tools.
 #
@@ -29,6 +29,7 @@ DEVICE_MAPPER_SOURCE=\
 	device_mapper/regex/parse_rx.c \
 	device_mapper/regex/ttree.c \
 	device_mapper/vdo/status.c \
+	device_mapper/vdo/vdo_reader.c \
 	device_mapper/vdo/vdo_target.c
 
 DEVICE_MAPPER_TARGET = device_mapper/libdevice-mapper.a

device_mapper/all.h

@@ -173,6 +173,16 @@ struct dm_names {
 	char name[];
 };
 
+struct dm_active_device {
+	struct dm_list list;
+	int major;
+	int minor;
+	char *name;	/* device name */
+
+	uint32_t event_nr; /* valid when DM_DEVICE_LIST_HAS_EVENT_NR is set */
+	char *uuid;	/* valid uuid when DM_DEVICE_LIST_HAS_UUID is set */
+};
+
 struct dm_versions {
 	uint32_t next;		/* Offset to next struct from start of this struct */
 	uint32_t version[3];
@@ -210,6 +220,25 @@ const char *dm_task_get_message_response(struct dm_task *dmt);
  */
 const char *dm_task_get_name(const struct dm_task *dmt);
 struct dm_names *dm_task_get_names(struct dm_task *dmt);
+/*
+ * Retrieve the list of devices and put them into easily accessible
+ * struct dm_active_device list elements.
+ * devs_features provides flag-set with used features so it's easy to check
+ * whether the kernel provides i.e. UUID info together with DM names
+ */
+#define DM_DEVICE_LIST_HAS_EVENT_NR	1
+#define DM_DEVICE_LIST_HAS_UUID		2
+int dm_task_get_device_list(struct dm_task *dmt, struct dm_list **devs_list,
+			    unsigned *devs_features);
+/*
+ * -1: no idea about uuid (not provided by DM_DEVICE_LIST ioctl)
+ *  0: uuid not present
+ *  1: listed and dm_active_device will be set for not NULL pointer
+ */
+int dm_device_list_find_by_uuid(struct dm_list *devs_list, const char *uuid,
+				const struct dm_active_device **dev);
+/* Release all associated memory with list of active DM devices */
+void dm_device_list_destroy(struct dm_list **devs_list);
 
 int dm_task_set_ro(struct dm_task *dmt);
 int dm_task_set_newname(struct dm_task *dmt, const char *newname);
@@ -953,7 +982,9 @@ struct writecache_settings {
 	uint32_t fua;
 	uint32_t nofua;
 	uint32_t cleaner;
-	uint32_t max_age;
+	uint32_t max_age;         /* in milliseconds */
+	uint32_t metadata_only;
+	uint32_t pause_writeback; /* in milliseconds */
 
 	/*
 	 * Allow an unrecognized key and its val to be passed to the kernel for
@@ -975,6 +1006,8 @@ struct writecache_settings {
 	unsigned nofua_set:1;
 	unsigned cleaner_set:1;
 	unsigned max_age_set:1;
+	unsigned metadata_only_set:1;
+	unsigned pause_writeback_set:1;
 };
 
 int dm_tree_node_add_writecache_target(struct dm_tree_node *node,
@@ -1020,6 +1053,7 @@ int dm_tree_node_add_integrity_target(struct dm_tree_node *node,
  */
 int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 				uint64_t size,
+				uint32_t vdo_version,
 				const char *vdo_pool_name,
 				const char *data_uuid,
 				uint64_t data_size,
@@ -1953,7 +1987,8 @@ struct dm_report_group;
 typedef enum {
 	DM_REPORT_GROUP_SINGLE,
 	DM_REPORT_GROUP_BASIC,
-	DM_REPORT_GROUP_JSON
+	DM_REPORT_GROUP_JSON,
+	DM_REPORT_GROUP_JSON_STD
 } dm_report_group_type_t;
 
 struct dm_report_group *dm_report_group_create(dm_report_group_type_t type, void *data);

device_mapper/ioctl/libdm-iface.c

@@ -139,7 +139,6 @@ static char *_align(char *ptr, unsigned int a)
 	return (char *) (((unsigned long) ptr + agn) & ~agn);
 }
 
-#ifdef DM_IOCTLS
 static unsigned _kernel_major = 0;
 static unsigned _kernel_minor = 0;
 static unsigned _kernel_release = 0;
@@ -182,6 +181,9 @@ int get_uname_version(unsigned *major, unsigned *minor, unsigned *release)
 
 	return 1;
 }
+
+#ifdef DM_IOCTLS
+
 /*
  * Set number to NULL to populate _dm_bitset - otherwise first
  * match is returned.
@@ -310,8 +312,13 @@ static int _create_control(const char *control, uint32_t major, uint32_t minor)
 	old_umask = umask(DM_CONTROL_NODE_UMASK);
 	if (mknod(control, S_IFCHR | S_IRUSR | S_IWUSR,
 		  MKDEV(major, minor)) < 0)  {
-		log_sys_error("mknod", control);
-		ret = 0;
+		if (errno != EEXIST) {
+			log_sys_error("mknod", control);
+			ret = 0;
+		} else if (_control_exists(control, major, minor) != 1) {
+			stack; /* Invalid control node created by parallel command ? */
+			ret = 0;
+		}
 	}
 	umask(old_umask);
 	(void) dm_prepare_selinux_context(NULL, 0);
@@ -616,8 +623,7 @@ int dm_check_version(void)
 int dm_cookie_supported(void)
 {
 	return (dm_check_version() &&
-		_dm_version >= 4 &&
-		_dm_version_minor >= 15);
+		((_dm_version == 4) ? _dm_version_minor >= 15 : _dm_version > 4));
 }
 
 static int _dm_inactive_supported(void)
@@ -755,6 +761,160 @@ struct dm_deps *dm_task_get_deps(struct dm_task *dmt)
 				   dmt->dmi.v4->data_start);
 }
 
+
+/*
+ * Round up the ptr to an 8-byte boundary.
+ * Follow kernel pattern.
+ */
+#define ALIGN_MASK 7
+static size_t _align_val(size_t val)
+{
+	return (val + ALIGN_MASK) & ~ALIGN_MASK;
+}
+static void *_align_ptr(void *ptr)
+{
+	return (void *)_align_val((size_t)ptr);
+}
+
+static int _check_has_event_nr(void) {
+	static int _has_event_nr = -1;
+
+	if (_has_event_nr < 0)
+		_has_event_nr = dm_check_version() &&
+			((_dm_version == 4) ?  _dm_version_minor >= 38 : _dm_version > 4);
+
+	return _has_event_nr;
+}
+
+struct dm_device_list {
+	struct dm_list list;
+	unsigned count;
+	unsigned features;
+	struct dm_hash_table *uuids;
+};
+
+int dm_task_get_device_list(struct dm_task *dmt, struct dm_list **devs_list,
+			    unsigned *devs_features)
+{
+	struct dm_names *names, *names1;
+	struct dm_active_device *dm_dev, *dm_new_dev;
+	struct dm_device_list *devs;
+	unsigned next = 0;
+	uint32_t *event_nr;
+	char *uuid_ptr;
+	size_t len;
+	int cnt = 0;
+
+	*devs_list = 0;
+	*devs_features = 0;
+
+	if ((names = dm_task_get_names(dmt)) && names->dev) {
+		names1 = names;
+		if (!names->name[0])
+			cnt = -1; /* -> cnt == 0 when no device is really present */
+		do {
+			names1 = (struct dm_names *)((char *) names1 + next);
+			next = names1->next;
+			++cnt;
+		} while (next);
+	}
+
+	if (!(devs = malloc(sizeof(*devs) + (cnt ? cnt * sizeof(*dm_dev) + (char*)names1 - (char*)names + 256 : 0))))
+		return_0;
+
+	dm_list_init(&devs->list);
+	devs->count = cnt;
+	devs->uuids = NULL;
+
+	if (!cnt) {
+		/* nothing in the list -> mark all features present */
+		*devs_features |= (DM_DEVICE_LIST_HAS_EVENT_NR | DM_DEVICE_LIST_HAS_UUID);
+		goto out; /* nothing else to do */
+	}
+
+	dm_dev = (struct dm_active_device *) (devs + 1);
+
+	do {
+		names = (struct dm_names *)((char *) names + next);
+
+		dm_dev->major = MAJOR(names->dev);
+		dm_dev->minor = MINOR(names->dev);
+		dm_dev->name = (char*)(dm_dev + 1);
+		dm_dev->event_nr = 0;
+		dm_dev->uuid = NULL;
+
+		len = strlen(names->name) + 1;
+		memcpy(dm_dev->name, names->name, len);
+
+		dm_new_dev = _align_ptr((char*)(dm_dev + 1) + len);
+		if (_check_has_event_nr()) {
+			/* Hash for UUIDs with some more bits to reduce colision count */
+			if (!devs->uuids && !(devs->uuids = dm_hash_create(cnt * 8))) {
+				free(devs);
+				return_0;
+			}
+
+			*devs_features |= DM_DEVICE_LIST_HAS_EVENT_NR;
+			event_nr = _align_ptr(names->name + len);
+			dm_dev->event_nr = event_nr[0];
+
+			if ((event_nr[1] & DM_NAME_LIST_FLAG_HAS_UUID)) {
+				*devs_features |= DM_DEVICE_LIST_HAS_UUID;
+				uuid_ptr = _align_ptr(event_nr + 2);
+				dm_dev->uuid = (char*) dm_new_dev;
+				len = strlen(uuid_ptr) + 1;
+				dm_new_dev = _align_ptr((char*)dm_new_dev + len);
+				memcpy(dm_dev->uuid, uuid_ptr, len);
+				if (!dm_hash_insert(devs->uuids, dm_dev->uuid, dm_dev))
+					return_0; // FIXME
+#if 0
+				log_debug("Active %s (%s) %d:%d event:%u",
+					  dm_dev->name, dm_dev->uuid,
+					  dm_dev->major, dm_dev->minor, dm_dev->event_nr);
+#endif
+			}
+		}
+
+		dm_list_add(&devs->list, &dm_dev->list);
+		dm_dev = dm_new_dev;
+		next = names->next;
+	} while (next);
+
+    out:
+	*devs_list = (struct dm_list *)devs;
+
+	return 1;
+}
+
+int dm_device_list_find_by_uuid(struct dm_list *devs_list, const char *uuid,
+				const struct dm_active_device **dev)
+{
+	struct dm_device_list *devs = (struct dm_device_list *) devs_list;
+	struct dm_active_device *dm_dev;
+
+	if (devs->uuids &&
+	    (dm_dev = dm_hash_lookup(devs->uuids, uuid))) {
+		if (dev)
+			*dev = dm_dev;
+		return 1;
+	}
+
+	return 0;
+}
+
+void dm_device_list_destroy(struct dm_list **devs_list)
+{
+	struct dm_device_list *devs = (struct dm_device_list *) *devs_list;
+
+	if (devs) {
+		if (devs->uuids)
+			dm_hash_destroy(devs->uuids);
+
+		free(devs);
+		*devs_list = NULL;
+	}
+}
+
 struct dm_names *dm_task_get_names(struct dm_task *dmt)
 {
 	return (struct dm_names *) (((char *) dmt->dmi.v4) +
@@ -1042,9 +1202,10 @@ static char *_add_target(struct target *t, char *out, char *end)
 	while (*pt)
 		if (*pt++ == '\\')
 			backslash_count++;
-	len = strlen(t->params) + backslash_count;
 
-	if ((out >= end) || (out + len + 1) >= end) {
+	len = strlen(t->params) + 1;
+
+	if ((out >= end) || (out + len + backslash_count) >= end) {
 		log_error("Ran out of memory building ioctl parameter");
 		return NULL;
 	}
@@ -1060,8 +1221,8 @@ static char *_add_target(struct target *t, char *out, char *end)
 		*out++ = '\0';
 	}
 	else {
-		strcpy(out, t->params);
-		out += len + 1;
+		memcpy(out, t->params, len);
+		out += len + backslash_count;
 	}
 
 	/* align next block */
@@ -1132,6 +1293,7 @@ static struct dm_ioctl *_flatten(struct dm_task *dmt, unsigned repeat_count)
 	struct target *t;
 	struct dm_target_msg *tmsg;
 	size_t len = sizeof(struct dm_ioctl);
+	size_t message_len = 0, newname_len = 0, geometry_len = 0;
 	char *b, *e;
 	int count = 0;
 
@@ -1192,14 +1354,20 @@ static struct dm_ioctl *_flatten(struct dm_task *dmt, unsigned repeat_count)
 		return NULL;
 	}
 
-	if (dmt->newname)
-		len += strlen(dmt->newname) + 1;
+	if (dmt->newname) {
+		newname_len = strlen(dmt->newname) + 1;
+		len += newname_len;
+	}
 
-	if (dmt->message)
-		len += sizeof(struct dm_target_msg) + strlen(dmt->message) + 1;
+	if (dmt->message) {
+		message_len = strlen(dmt->message) + 1;
+		len += sizeof(struct dm_target_msg) + message_len;
+	}
 
-	if (dmt->geometry)
-		len += strlen(dmt->geometry) + 1;
+	if (dmt->geometry) {
+		geometry_len = strlen(dmt->geometry) + 1;
+		len += geometry_len;
+	}
 
 	/*
 	 * Give len a minimum size so that we have space to store
@@ -1321,16 +1489,16 @@ static struct dm_ioctl *_flatten(struct dm_task *dmt, unsigned repeat_count)
 				goto_bad;
 
 	if (dmt->newname)
-		strcpy(b, dmt->newname);
+		memcpy(b, dmt->newname, newname_len);
 
 	if (dmt->message) {
 		tmsg = (struct dm_target_msg *) b;
 		tmsg->sector = dmt->sector;
-		strcpy(tmsg->message, dmt->message);
+		memcpy(tmsg->message, dmt->message, message_len);
 	}
 
 	if (dmt->geometry)
-		strcpy(b, dmt->geometry);
+		memcpy(b, dmt->geometry, geometry_len);
 
 	return dmi;
 
@@ -1441,8 +1609,7 @@ static int _udev_complete(struct dm_task *dmt)
 static int _check_uevent_generated(struct dm_ioctl *dmi)
 {
 	if (!dm_check_version() ||
-	    _dm_version < 4 ||
-	    _dm_version_minor < 17)
+	    ((_dm_version == 4) ? _dm_version_minor < 17 : _dm_version < 4))
 		/* can't check, assume uevent is generated */
 		return 1;
 
@@ -1808,23 +1975,34 @@ static int _do_dm_ioctl_unmangle_string(char *str, const char *str_name,
 static int _dm_ioctl_unmangle_names(int type, struct dm_ioctl *dmi)
 {
 	char buf[DM_NAME_LEN];
-	struct dm_names *names;
+	char buf_uuid[DM_UUID_LEN];
+	struct dm_name_list *names;
 	unsigned next = 0;
 	char *name;
 	int r = 1;
+	uint32_t *event_nr;
+	char *uuid_ptr;
+	dm_string_mangling_t mangling_mode = dm_get_name_mangling_mode();
 
 	if ((name = dmi->name))
-		r = _do_dm_ioctl_unmangle_string(name, "name", buf, sizeof(buf),
-						 dm_get_name_mangling_mode());
+		r &= _do_dm_ioctl_unmangle_string(name, "name", buf, sizeof(buf),
+						  mangling_mode);
 
 	if (type == DM_DEVICE_LIST &&
-	    ((names = ((struct dm_names *) ((char *)dmi + dmi->data_start)))) &&
+	    ((names = ((struct dm_name_list *) ((char *)dmi + dmi->data_start)))) &&
 	    names->dev) {
 		do {
-			names = (struct dm_names *)((char *) names + next);
-			r = _do_dm_ioctl_unmangle_string(names->name, "name",
-							 buf, sizeof(buf),
-							 dm_get_name_mangling_mode());
+			names = (struct dm_name_list *)((char *) names + next);
+			event_nr = _align_ptr(names->name + strlen(names->name) + 1);
+			r &= _do_dm_ioctl_unmangle_string(names->name, "name",
+							  buf, sizeof(buf), mangling_mode);
+			/* Unmangle also UUID within same loop */
+			if (_check_has_event_nr() &&
+			    (event_nr[1] & DM_NAME_LIST_FLAG_HAS_UUID)) {
+				uuid_ptr = _align_ptr(event_nr + 2);
+				r &= _do_dm_ioctl_unmangle_string(uuid_ptr, "UUID", buf_uuid,
+								  sizeof(buf_uuid), mangling_mode);
+			}
 			next = names->next;
 		} while (next);
 	}

device_mapper/libdm-deptree.c

@@ -214,6 +214,7 @@ struct load_segment {
 	uint32_t device_id;		/* Thin */
 
 	// VDO params
+	uint32_t vdo_version;		/* VDO - version of target table line */
 	struct dm_tree_node *vdo_data;  /* VDO */
 	struct dm_vdo_target_params vdo_params; /* VDO */
 	const char *vdo_name;           /* VDO - device name is ALSO passed as table arg */
@@ -293,6 +294,10 @@ struct load_properties {
 	unsigned send_messages;
 	/* Skip suspending node's children, used when sending messages to thin-pool */
 	int skip_suspend;
+
+	/* Suspend and Resume siblings after node activation with udev flags*/
+	unsigned reactivate_siblings;
+	uint16_t reactivate_udev_flags;
 };
 
 /* Two of these used to join two nodes with uses and used_by. */
@@ -2029,6 +2034,68 @@ static int _rename_conflict_exists(struct dm_tree_node *parent,
 	return 0;
 }
 
+/*
+ * Reactivation of sibling nodes
+ *
+ * Function is used when activating origin and its thick snapshots
+ * to ensure udev is processing first the origin LV and all the
+ * snapshot LVs are processed afterwards.
+ */
+static int _reactivate_siblings(struct dm_tree_node *dnode,
+				const char *uuid_prefix,
+				size_t uuid_prefix_len)
+{
+	struct dm_tree_node *child;
+	const char *uuid;
+	void *handle = NULL;
+	int r = 1;
+
+	/* Wait for udev before reactivating siblings */
+	if (!dm_udev_wait(dm_tree_get_cookie(dnode)))
+		stack;
+
+	dm_tree_set_cookie(dnode, 0);
+
+	while ((child = dm_tree_next_child(&handle, dnode, 0))) {
+		if (child->props.reactivate_siblings) {
+			/* Skip 'leading' device in this group, marked with flag */
+			child->props.reactivate_siblings = 0;
+			continue;
+		}
+
+		if (!(uuid = dm_tree_node_get_uuid(child))) {
+			stack;
+			continue;
+		}
+
+		if (!_uuid_prefix_matches(uuid, uuid_prefix, uuid_prefix_len))
+			continue;
+
+		if (!_suspend_node(child->name, child->info.major, child->info.minor,
+				   child->dtree->skip_lockfs,
+				   child->dtree->no_flush, &child->info)) {
+			log_error("Unable to suspend %s (" FMTu32
+				  ":" FMTu32 ")", child->name,
+				  child->info.major, child->info.minor);
+			r = 0;
+			continue;
+		}
+		if (!_resume_node(child->name, child->info.major, child->info.minor,
+				  child->props.read_ahead, child->props.read_ahead_flags,
+				  &child->info, &child->dtree->cookie,
+				  child->props.reactivate_udev_flags, // use these flags
+				  child->info.suspended)) {
+			log_error("Failed to suspend %s (" FMTu32
+				  ":" FMTu32 ")", child->name,
+				  child->info.major, child->info.minor);
+			r = 0;
+			continue;
+		}
+	}
+
+	return r;
+}
+
 int dm_tree_activate_children(struct dm_tree_node *dnode,
 				 const char *uuid_prefix,
 				 size_t uuid_prefix_len)
@@ -2039,7 +2106,7 @@ int dm_tree_activate_children(struct dm_tree_node *dnode,
 	struct dm_tree_node *child = dnode;
 	const char *name;
 	const char *uuid;
-	int priority;
+	int priority, next_priority;
 
 	/* Activate children first */
 	while ((child = dm_tree_next_child(&handle, dnode, 0))) {
@@ -2057,12 +2124,16 @@ int dm_tree_activate_children(struct dm_tree_node *dnode,
 	}
 
 	handle = NULL;
-
 	for (priority = 0; priority < 3; priority++) {
 		awaiting_peer_rename = 0;
+		next_priority = 0;
 		while ((child = dm_tree_next_child(&handle, dnode, 0))) {
-			if (priority != child->activation_priority)
+			if (priority != child->activation_priority) {
+				if ((next_priority < child->activation_priority) &&
+				    (child->activation_priority > priority))
+					next_priority = child->activation_priority;
 				continue;
+			}
 
 			if (!(uuid = dm_tree_node_get_uuid(child))) {
 				stack;
@@ -2117,9 +2188,16 @@ int dm_tree_activate_children(struct dm_tree_node *dnode,
 			if (r && (child->props.send_messages > 1) &&
 			    !(r = _node_send_messages(child, uuid_prefix, uuid_prefix_len, 1)))
 				stack;
+
+			/* Reactivate only for fresh activated origin */
+			if (r && child->props.reactivate_siblings &&
+			    (!(r = _reactivate_siblings(dnode, uuid_prefix, uuid_prefix_len))))
+				stack;
 		}
 		if (awaiting_peer_rename)
 			priority--; /* redo priority level */
+		else if (!next_priority)
+			break;  /* no more work, higher priority was not found in the chain */
 	}
 
 	return r;
@@ -2678,6 +2756,10 @@ static int _writecache_emit_segment_line(struct dm_task *dmt,
 		count += 1;
 	if (seg->writecache_settings.max_age_set)
 		count += 2;
+	if (seg->writecache_settings.metadata_only_set)
+		count += 1;
+	if (seg->writecache_settings.pause_writeback_set)
+		count += 2;
 	if (seg->writecache_settings.new_key)
 		count += 2;
 
@@ -2729,6 +2811,14 @@ static int _writecache_emit_segment_line(struct dm_task *dmt,
 		EMIT_PARAMS(pos, " max_age %u", seg->writecache_settings.max_age);
 	}
 
+	if (seg->writecache_settings.metadata_only_set) {
+		EMIT_PARAMS(pos, " metadata_only");
+	}
+
+	if (seg->writecache_settings.pause_writeback_set) {
+		EMIT_PARAMS(pos, " pause_writeback %u", seg->writecache_settings.pause_writeback);
+	}
+
 	if (seg->writecache_settings.new_key) {
 		EMIT_PARAMS(pos, " %s %s",
 			seg->writecache_settings.new_key,
@@ -2849,13 +2939,18 @@ static int _thin_pool_emit_segment_line(struct dm_task *dmt,
 	return 1;
 }
 
-static int _vdo_emit_segment_line(struct dm_task *dmt,
+static int _vdo_emit_segment_line(struct dm_task *dmt, uint32_t major, uint32_t minor,
 				  struct load_segment *seg,
 				  char *params, size_t paramsize)
 {
 	int pos = 0;
 	char data[DM_FORMAT_DEV_BUFSIZE];
 	char data_dev[128]; // for /dev/dm-XXXX
+	uint64_t logical_blocks;
+	struct dm_task *vdo_dmt;
+	uint64_t start, length = 0;
+	char *type = NULL;
+	char *vdo_params = NULL;
 
 	if (!_build_dev_string(data, sizeof(data), seg->vdo_data))
 		return_0;
@@ -2865,18 +2960,59 @@ static int _vdo_emit_segment_line(struct dm_task *dmt,
 		return 0;
 	}
 
-	EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s "
-		    "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u",
-		    data_dev,
-		    seg->vdo_data_size / 8, // this parameter is in 4K units
-		    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
-		    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
-		    seg->vdo_params.block_map_era_length,
-		    seg->vdo_params.use_metadata_hints ? "on" : "off" ,
-		    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" :
-			(seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" :
-			(seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy
-		    seg->vdo_name,
+	/*
+	 * If there is already running VDO target, read 'existing' virtual size out of table line
+	 * and avoid reading it them from VDO metadata device
+	 *
+	 * NOTE: ATM VDO virtual size can be ONLY extended thus it's simple to recongnize 'right' size.
+	 * However if there would be supported also reduction, this check would need to check range.
+	 */
+	if ((vdo_dmt = dm_task_create(DM_DEVICE_TABLE))) {
+		if (dm_task_set_major(vdo_dmt, major) &&
+		    dm_task_set_minor(vdo_dmt, minor) &&
+		    dm_task_run(vdo_dmt)) {
+			(void) dm_get_next_target(vdo_dmt, NULL, &start, &length, &type, &vdo_params);
+			if (!type || strcmp(type, "vdo"))
+				length = 0;
+		}
+
+		dm_task_destroy(vdo_dmt);
+	}
+
+	if (!length && dm_vdo_parse_logical_size(data_dev, &logical_blocks))
+		length = logical_blocks * 8;
+
+	if (seg->size < length) {
+		log_debug_activation("Correcting VDO virtual volume size from " FMTu64  " to " FMTu64 ".",
+				     seg->size, length);
+		seg->size = length;
+	}
+
+	if (seg->vdo_version < 4) {
+		EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s ",
+			    data_dev,
+			    seg->vdo_data_size / 8, // this parameter is in 4K units
+			    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
+			    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
+			    seg->vdo_params.block_map_era_length,
+			    seg->vdo_params.use_metadata_hints ? "on" : "off" ,
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" :
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" :
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy
+			    seg->vdo_name);
+	} else {
+		EMIT_PARAMS(pos, "V4 %s " FMTu64 " %u " FMTu64 " %u "
+			    "deduplication %s compression %s ",
+			    data_dev,
+			    seg->vdo_data_size / 8, // this parameter is in 4K units
+			    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
+			    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
+			    seg->vdo_params.block_map_era_length,
+			    seg->vdo_params.use_deduplication ? "on" : "off",
+			    seg->vdo_params.use_compression ? "on" : "off");
+	}
+
+	EMIT_PARAMS(pos, "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u",
 		    seg->vdo_params.max_discard,
 		    seg->vdo_params.ack_threads,
 		    seg->vdo_params.bio_threads,
@@ -2951,7 +3087,7 @@ static int _emit_segment_line(struct dm_task *dmt, uint32_t major,
 		EMIT_PARAMS(pos, "%u %u ", seg->area_count, seg->stripe_size);
 		break;
 	case SEG_VDO:
-		if (!_vdo_emit_segment_line(dmt, seg, params, paramsize))
+		if (!_vdo_emit_segment_line(dmt, major, minor, seg, params, paramsize))
 		      return_0;
 		break;
 	case SEG_CRYPT:
@@ -3390,6 +3526,10 @@ int dm_tree_node_add_snapshot_origin_target(struct dm_tree_node *dnode,
 	/* Resume snapshot origins after new snapshots */
 	dnode->activation_priority = 1;
 
+	if (!dnode->info.exists)
+		/* Reactivate siblings for this origin after being resumed */
+		dnode->props.reactivate_siblings = 1;
+
 	/*
 	 * Don't resume the origin immediately in case it is a non-trivial 
 	 * target that must not be active more than once concurrently!
@@ -3452,6 +3592,20 @@ static int _add_snapshot_target(struct dm_tree_node *node,
 			/* Resume merging snapshot after snapshot-merge */
 			seg->merge->activation_priority = 2;
 		}
+	} else if (!origin_node->info.exists) {
+		/* Keep original udev_flags for reactivation. */
+		node->props.reactivate_udev_flags = node->udev_flags;
+
+		/* Reactivation is needed if the origin's -real device is not in DM table.
+		 * For this case after the resume of its origin LV we resume its snapshots
+		 * with updated udev_flags to completely avoid udev scanning for the first resume.
+		 * Reactivation then resumes snapshots with original udev_flags.
+		 */
+		node->udev_flags |= DM_SUBSYSTEM_UDEV_FLAG0 |
+			DM_UDEV_DISABLE_DISK_RULES_FLAG |
+			DM_UDEV_DISABLE_OTHER_RULES_FLAG;
+		log_debug_activation("Using udev_flags 0x%x for activation of %s.",
+				     node->udev_flags, node->name);
 	}
 
 	return 1;
@@ -4323,6 +4477,7 @@ void dm_tree_node_set_callback(struct dm_tree_node *dnode,
 
 int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 				uint64_t size,
+				uint32_t vdo_version,
 				const char *vdo_pool_name,
 				const char *data_uuid,
 				uint64_t data_size,
@@ -4344,11 +4499,13 @@ int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 	if (!_link_tree_nodes(node, seg->vdo_data))
 		return_0;
 
+	seg->vdo_version = vdo_version;
 	seg->vdo_params = *vtp;
 	seg->vdo_name = vdo_pool_name;
 	seg->vdo_data_size = data_size;
 
-	node->props.send_messages = 2;
+	if (seg->vdo_version < 4)
+		node->props.send_messages = 2;
 
 	return 1;
 }

device_mapper/libdm-report.c

@@ -384,172 +384,246 @@ int dm_report_field_percent(struct dm_report *rh,
 	return 1;
 }
 
-struct str_list_sort_value_item {
+struct pos_len {
 	unsigned pos;
 	size_t len;
 };
 
+struct str_pos_len {
+	const char *str;
+	struct pos_len item;
+};
+
 struct str_list_sort_value {
 	const char *value;
-	struct str_list_sort_value_item *items;
+	struct pos_len *items;
 };
 
-struct str_list_sort_item {
-	const char *str;
-	struct str_list_sort_value_item item;
-};
-
-static int _str_list_sort_item_cmp(const void *a, const void *b)
+static int _str_sort_cmp(const void *a, const void *b)
 {
-	const struct str_list_sort_item *slsi_a = (const struct str_list_sort_item *) a;
-	const struct str_list_sort_item *slsi_b = (const struct str_list_sort_item *) b;
-
-	return strcmp(slsi_a->str, slsi_b->str);
+	return strcmp(((const struct str_pos_len *) a)->str, ((const struct str_pos_len *) b)->str);
 }
 
+#define FIELD_STRING_LIST_DEFAULT_DELIMITER ","
+
 static int _report_field_string_list(struct dm_report *rh,
 				     struct dm_report_field *field,
 				     const struct dm_list *data,
 				     const char *delimiter,
-				     int sort)
+				     int sort_repstr)
 {
-	static const char _string_list_grow_object_failed_msg[] = "dm_report_field_string_list: dm_pool_grow_object_failed";
-	struct str_list_sort_value *sort_value = NULL;
-	unsigned int list_size, pos, i;
-	struct str_list_sort_item *arr = NULL;
+	static const char _error_msg_prefix[] = "_report_field_string_list: ";
+	unsigned int list_size, i, pos;
+	struct str_pos_len *arr = NULL;
 	struct dm_str_list *sl;
-	size_t delimiter_len, len;
-	void *object;
+	size_t delimiter_len, repstr_str_len, repstr_size;
+	char *repstr = NULL;
+	struct pos_len *repstr_extra;
+	struct str_list_sort_value *sortval = NULL;
 	int r = 0;
 
-	if (!(sort_value = dm_pool_zalloc(rh->mem, sizeof(struct str_list_sort_value)))) {
-		log_error("dm_report_field_string_list: dm_pool_zalloc failed for sort_value");
-		return 0;
-	}
+	/*
+	 * The 'field->report_string' has 2 parts:
+	 *
+	 *    - string representing the whole string list
+	 *      (terminated by '\0' at its end as usual)
+	 *
+	 *    - extra info beyond the end of the string representing
+	 *      position and length of each list item within the
+	 *      field->report_string (array of 'struct pos_len')
+	 *
+	 * We can use the extra info to unambiguously identify list items,
+	 * the delimiter is not enough here as it's not assured it won't appear
+	 * in list item itself. We will make use of this extra info in case
+	 * we need to apply further formatting to the list in dm_report_output
+	 * where the pure field->report_string is not enough for printout.
+	 *
+	 *
+	 * The 'field->sort_value' contains a value of type 'struct
+	 * str_list_sort_value' ('sortval'). This one has a pointer to the
+	 * 'field->report_string' string ('sortval->value') and info
+	 * about position and length of each list item within the string
+	 * (array of 'struct pos_len').
+	 *
+	 *
+	 * The 'field->report_string' is either in sorted or unsorted form,
+	 * depending on 'sort_repstr' arg.
+	 *
+	 * The 'field->sort_value.items' is always in sorted form because
+	 * we need that for effective sorting and selection.
+	 *
+	 * If 'field->report_string' is sorted, then field->report_string
+	 * and field->sort_value.items share the same array of
+	 * 'struct pos_len' (because they're both sorted the same way),
+	 * otherwise, each one has its own array.
+	 *
+	 * The very first item in the array of 'struct pos_len' is always
+	 * a pair denoting '[list_size,strlen(field->report_string)]'. The
+	 * rest of items denote start and lenght of each item in the list.
+         *
+	 *
+	 * For example, if we have a list with "abc", "xy", "defgh"
+	 * as input and delimiter is ",", we end up with either:
+	 *
+	 *  A) if we don't want the report string sorted ('sort_repstr == 0'):
+	 *
+	 *    - field->report_string = repstr
+	 *
+	 *       repstr      repstr_extra
+	 *         |             |
+	 *         V             V
+	 *         abc,xy,defgh\0{[3,12],[0,3],[4,2],[7,5]}
+	 *         |____________||________________________|
+	 *             string     array of struct pos_len
+	 *                        |____||________________|
+	 *                        #items      items
+	 *
+	 *    - field->sort_value = sortval
+	 *
+	 *       sortval->value = repstr
+	 *       sortval->items = {[3,12],[0,3],[7,5],[4,2]}
+	 *                         (that is 'abc,defgh,xy')
+	 *
+	 *
+	 *  B) if we want the report string sorted ('sort_repstr == 1'):
+	 *
+	 *    - field->report_string = repstr
+	 *
+	 *       repstr     repstr_extra
+	 *         |             |
+	 *         V             V
+	 *         abc,defgh,xy\0{[3,12],[0,3],[4,5],[10,2]}
+	 *         |____________||________________________|
+	 *             string     array of struct pos_len
+	 *                        |____||________________|
+	 *                        #items      items
+	 *
+	 *     - field->sort_value = sortval
+	 *
+	 *       sortval->value = repstr
+	 *       sortval->items = repstr_extra
+	 *                       (that is 'abc,defgh,xy')
+	 */
 
+	if (!delimiter)
+		delimiter = FIELD_STRING_LIST_DEFAULT_DELIMITER;
+	delimiter_len = strlen(delimiter);
 	list_size = dm_list_size(data);
 
-	/*
-	 * Sort value stores the pointer to the report_string and then
-	 * position and length for each list element withing the report_string.
-	 * The first element stores number of elements in 'len' (therefore
-	 * list_size + 1 is used below for the extra element).
-	 * For example, with this input:
-	 *   sort = 0;  (we don't want to report sorted)
-	 *   report_string = "abc,xy,defgh";  (this is reported)
-	 *
-	 * ...we end up with:
-	 *   sort_value->value = report_string; (we'll use the original report_string for indices)
-	 *   sort_value->items[0] = {0,3};  (we have 3 items)
-	 *   sort_value->items[1] = {0,3};  ("abc")
-	 *   sort_value->items[2] = {7,5};  ("defgh")
-	 *   sort_value->items[3] = {4,2};  ("xy")
-	 *
-	 *   The items alone are always sorted while in report_string they can be
-	 *   sorted or not (based on "sort" arg) - it depends on how we prefer to
-	 *   display the list. Having items sorted internally helps with searching
-	 *   through them.
-	 */
-	if (!(sort_value->items = dm_pool_zalloc(rh->mem, (list_size + 1) * sizeof(struct str_list_sort_value_item)))) {
-		log_error("dm_report_fiel_string_list: dm_pool_zalloc failed for sort value items");
+	if (!(sortval = dm_pool_alloc(rh->mem, sizeof(struct str_list_sort_value)))) {
+		log_error("%s failed to allocate sort value structure", _error_msg_prefix);
 		goto out;
 	}
-	sort_value->items[0].len = list_size;
 
 	/* zero items */
-	if (!list_size) {
-		sort_value->value = field->report_string = "";
-		field->sort_value = sort_value;
+	if (list_size == 0) {
+		field->report_string = sortval->value = "";
+		sortval->items = NULL;
+		field->sort_value = sortval;
 		return 1;
 	}
 
 	/* one item */
 	if (list_size == 1) {
 		sl = (struct dm_str_list *) dm_list_first(data);
-		if (!sl ||
-		    !(sort_value->value = field->report_string = dm_pool_strdup(rh->mem, sl->str))) {
-			log_error("dm_report_field_string_list: dm_pool_strdup failed");
+
+		repstr_str_len = strlen(sl->str);
+		repstr_size = repstr_str_len + 1 + (2 * sizeof(struct pos_len));
+
+		if (!(repstr = dm_pool_alloc(rh->mem, repstr_size))) {
+			log_error("%s failed to allocate report string structure", _error_msg_prefix);
 			goto out;
 		}
-		sort_value->items[1].pos = 0;
-		sort_value->items[1].len = strlen(sl->str);
-		field->sort_value = sort_value;
+		repstr_extra = (struct pos_len *) (repstr + repstr_str_len + 1);
+
+		memcpy(repstr, sl->str, repstr_str_len + 1);
+		memcpy(repstr_extra, &((struct pos_len) {.pos = 1, .len = repstr_str_len}), sizeof(struct pos_len));
+		memcpy(repstr_extra + 1, &((struct pos_len) {.pos = 0, .len = repstr_str_len}), sizeof(struct pos_len));
+
+		sortval->value = field->report_string = repstr;
+		sortval->items = repstr_extra;
+		field->sort_value = sortval;
 		return 1;
 	}
 
-	/* more than one item - sort the list */
-	if (!(arr = malloc(sizeof(struct str_list_sort_item) * list_size))) {
-		log_error("dm_report_field_string_list: malloc failed");
+	/* more than one item - allocate temporary array for string list items for further processing */
+	if (!(arr = malloc(list_size * sizeof(struct str_pos_len)))) {
+		log_error("%s failed to allocate temporary array for processing", _error_msg_prefix);
 		goto out;
 	}
 
-	if (!(dm_pool_begin_object(rh->mem, 256))) {
-		log_error(_string_list_grow_object_failed_msg);
-		goto out;
-	}
-
-	if (!delimiter)
-		delimiter = ",";
-	delimiter_len = strlen(delimiter);
-
-	i = pos = 0;
+	i = 0;
+	repstr_size = 0;
 	dm_list_iterate_items(sl, data) {
 		arr[i].str = sl->str;
-		if (!sort) {
-			/* sorted outpud not required - report the list as it is */
-			len = strlen(sl->str);
-			if (!dm_pool_grow_object(rh->mem, arr[i].str, len) ||
-			    (i+1 != list_size && !dm_pool_grow_object(rh->mem, delimiter, delimiter_len))) {
-				log_error(_string_list_grow_object_failed_msg);
-				goto out;
-			}
-			arr[i].item.pos = pos;
-			arr[i].item.len = len;
-			pos = i+1 == list_size ? pos+len : pos+len+delimiter_len;
-		}
+		repstr_size += (arr[i].item.len = strlen(sl->str));
 		i++;
 	}
 
-	qsort(arr, i, sizeof(struct str_list_sort_item), _str_list_sort_item_cmp);
+	/*
+	 * At this point, repstr_size contains sum of lengths of all string list items.
+	 * Now, add these to the repstr_size:
+	 *
+	 * 	--> sum of character count used by all delimiters: + ((list_size - 1) * delimiter_len)
+	 *
+	 * 	--> '\0' used at the end of the string list: + 1
+	 *
+	 * 	--> sum of structures used to keep info about pos and length of each string list item:
+	 * 	    [0, <list_size>] [<pos1>,<size1>] [<pos2>,<size2>] ...
+	 * 	    That is: + ((list_size + 1) * sizeof(struct pos_len))
+	 */
+	repstr_size += ((list_size - 1) * delimiter_len);
+	repstr_str_len = repstr_size;
+	repstr_size += 1 + ((list_size + 1) * sizeof(struct pos_len));
 
-	for (i = 0, pos = 0; i < list_size; i++) {
-		if (sort) {
-			/* sorted output required - report the list as sorted */
-			len = strlen(arr[i].str);
-			if (!dm_pool_grow_object(rh->mem, arr[i].str, len) ||
-			    (i+1 != list_size && !dm_pool_grow_object(rh->mem, delimiter, delimiter_len))) {
-				log_error(_string_list_grow_object_failed_msg);
-				goto out;
-			}
-			/*
-			 * Save position and length of the string
-			 * element in report_string for sort_value.
-			 * Use i+1 here since items[0] stores list size!!!
-			 */
-			sort_value->items[i+1].pos = pos;
-			sort_value->items[i+1].len = len;
-			pos = i+1 == list_size ? pos+len : pos+len+delimiter_len;
-		} else {
-			sort_value->items[i+1].pos = arr[i].item.pos;
-			sort_value->items[i+1].len = arr[i].item.len;
-		}
-	}
+	if (sort_repstr)
+		qsort(arr, list_size, sizeof(struct str_pos_len), _str_sort_cmp);
 
-	if (!dm_pool_grow_object(rh->mem, "\0", 1)) {
-		log_error(_string_list_grow_object_failed_msg);
+	if (!(repstr = dm_pool_alloc(rh->mem, repstr_size))) {
+		log_error("%s failed to allocate report string structure", _error_msg_prefix);
 		goto out;
 	}
+	repstr_extra = (struct pos_len *) (repstr + repstr_str_len + 1);
 
-	object = dm_pool_end_object(rh->mem);
-	sort_value->value = object;
-	field->sort_value = sort_value;
-	field->report_string = object;
+	memcpy(repstr_extra, &(struct pos_len) {.pos = list_size, .len = repstr_str_len}, sizeof(struct pos_len));
+	for (i = 0, pos = 0; i < list_size; i++) {
+		arr[i].item.pos = pos;
+
+		memcpy(repstr + pos, arr[i].str, arr[i].item.len);
+		memcpy(repstr_extra + i + 1, &arr[i].item, sizeof(struct pos_len));
+
+		pos += arr[i].item.len;
+		if (i + 1 < list_size) {
+			memcpy(repstr + pos, delimiter, delimiter_len);
+			pos += delimiter_len;
+		}
+	}
+	*(repstr + pos) = '\0';
+
+	sortval->value = repstr;
+	if (sort_repstr)
+		sortval->items = repstr_extra;
+	else {
+		if (!(sortval->items = dm_pool_alloc(rh->mem, (list_size + 1) * sizeof(struct pos_len)))) {
+			log_error("%s failed to allocate array of items inside sort value structure",
+				  _error_msg_prefix);
+			goto out;
+		}
+
+		qsort(arr, list_size, sizeof(struct str_pos_len), _str_sort_cmp);
+
+		sortval->items[0] = (struct pos_len) {.pos = list_size, .len = repstr_str_len};
+		for (i = 0; i < list_size; i++)
+			sortval->items[i+1] = arr[i].item;
+	}
+
+	field->report_string = repstr;
+	field->sort_value = sortval;
 	r = 1;
 out:
-	if (!r && sort_value)
-		dm_pool_free(rh->mem, sort_value);
+	if (!r && sortval)
+		dm_pool_free(rh->mem, sortval);
 	free(arr);
-
 	return r;
 }
 
@@ -1688,7 +1762,7 @@ static int _cmp_field_string_list_strict_all(const struct str_list_sort_value *v
 	struct dm_str_list *sel_item;
 	unsigned int i = 1;
 
-	if (!val->items[0].len) {
+	if (!val->items) {
 		if (sel_list_size == 1) {
 			/* match blank string list with selection defined as blank string only */
 			sel_item = dm_list_item(dm_list_first(&sel->str_list.list), struct dm_str_list);
@@ -1698,7 +1772,7 @@ static int _cmp_field_string_list_strict_all(const struct str_list_sort_value *v
 	}
 
 	/* if item count differs, it's clear the lists do not match */
-	if (val->items[0].len != sel_list_size)
+	if (val->items[0].pos != sel_list_size)
 		return 0;
 
 	/* both lists are sorted so they either match 1:1 or not */
@@ -1721,7 +1795,7 @@ static int _cmp_field_string_list_subset_all(const struct str_list_sort_value *v
 	unsigned int i, last_found = 1;
 	int r = 0;
 
-	if (!val->items[0].len) {
+	if (!val->items) {
 		if (sel_list_size == 1) {
 			/* match blank string list with selection defined as blank string only */
 			sel_item = dm_list_item(dm_list_first(&sel->str_list.list), struct dm_str_list);
@@ -1733,7 +1807,7 @@ static int _cmp_field_string_list_subset_all(const struct str_list_sort_value *v
 	/* check selection is a subset of the value */
 	dm_list_iterate_items(sel_item, &sel->str_list.list) {
 		r = 0;
-		for (i = last_found; i <= val->items[0].len; i++) {
+		for (i = last_found; i <= val->items[0].pos; i++) {
 			if ((strlen(sel_item->str) == val->items[i].len) &&
 			    !strncmp(sel_item->str, val->value + val->items[i].pos, val->items[i].len)) {
 				last_found = i;
@@ -1755,7 +1829,7 @@ static int _cmp_field_string_list_any(const struct str_list_sort_value *val,
 	unsigned int i;
 
 	/* match blank string list with selection that contains blank string */
-	if (!val->items[0].len) {
+	if (!val->items) {
 		dm_list_iterate_items(sel_item, &sel->str_list.list) {
 			if (!strcmp(sel_item->str, ""))
 				return 1;
@@ -1768,7 +1842,7 @@ static int _cmp_field_string_list_any(const struct str_list_sort_value *val,
 		 * TODO: Optimize this so we don't need to compare the whole lists' content.
 		 *       Make use of the fact that the lists are sorted!
 		 */
-		for (i = 1; i <= val->items[0].len; i++) {
+		for (i = 1; i <= val->items[0].pos; i++) {
 			if ((strlen(sel_item->str) == val->items[i].len) &&
 			    !strncmp(sel_item->str, val->value + val->items[i].pos, val->items[i].len))
 				return 1;
@@ -4360,6 +4434,7 @@ static int _sort_rows(struct dm_report *rh)
 #define JSON_ARRAY_START       "["
 #define JSON_ARRAY_END         "]"
 #define JSON_ESCAPE_CHAR       "\\"
+#define JSON_NULL              "null"
 
 #define UNABLE_TO_EXTEND_OUTPUT_LINE_MSG "dm_report: Unable to extend output line"
 
@@ -4369,38 +4444,42 @@ static int _is_basic_report(struct dm_report *rh)
 	       (rh->group_item->group->type == DM_REPORT_GROUP_BASIC);
 }
 
+static int _is_json_std_report(struct dm_report *rh)
+{
+	return rh->group_item &&
+	       rh->group_item->group->type == DM_REPORT_GROUP_JSON_STD;
+}
+
 static int _is_json_report(struct dm_report *rh)
 {
 	return rh->group_item &&
-	       (rh->group_item->group->type == DM_REPORT_GROUP_JSON);
+	       (rh->group_item->group->type == DM_REPORT_GROUP_JSON ||
+		rh->group_item->group->type == DM_REPORT_GROUP_JSON_STD);
 }
 
-/*
- * Produce report output
- */
-static int _output_field(struct dm_report *rh, struct dm_report_field *field)
+static int _is_pure_numeric_field(struct dm_report_field *field)
+{
+	return field->props->flags & (DM_REPORT_FIELD_TYPE_NUMBER | DM_REPORT_FIELD_TYPE_PERCENT);
+}
+
+static const char *_get_field_id(struct dm_report *rh, struct dm_report_field *field)
 {
 	const struct dm_report_field_type *fields = field->props->implicit ? _implicit_report_fields
 									   : rh->fields;
+
+	return fields[field->props->field_num].id;
+}
+
+static int _output_field_basic_fmt(struct dm_report *rh, struct dm_report_field *field)
+{
 	char *field_id;
 	int32_t width;
 	uint32_t align;
-	const char *repstr;
-	const char *p1_repstr, *p2_repstr;
 	char *buf = NULL;
 	size_t buf_size = 0;
 
-	if (_is_json_report(rh)) {
-		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
-		    !dm_pool_grow_object(rh->mem, fields[field->props->field_num].id, 0) ||
-		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
-		    !dm_pool_grow_object(rh->mem, JSON_PAIR, 1) ||
-		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-			log_error("dm_report: Unable to extend output line");
-			return 0;
-		}
-	} else if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
-		if (!(field_id = strdup(fields[field->props->field_num].id))) {
+	if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
+		if (!(field_id = strdup(_get_field_id(rh, field)))) {
 			log_error("dm_report: Failed to copy field name");
 			return 0;
 		}
@@ -4431,43 +4510,14 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 		}
 	}
 
-	repstr = field->report_string;
-	width = field->props->width;
-	if (!(rh->flags & DM_REPORT_OUTPUT_ALIGNED)) {
-		if (_is_json_report(rh)) {
-			/* Escape any JSON_QUOTE that may appear in reported string. */
-			p1_repstr = repstr;
-			while ((p2_repstr = strstr(p1_repstr, JSON_QUOTE))) {
-				if (p2_repstr > p1_repstr) {
-					if (!dm_pool_grow_object(rh->mem, p1_repstr, p2_repstr - p1_repstr)) {
-						log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-						return 0;
-					}
-				}
-				if (!dm_pool_grow_object(rh->mem, JSON_ESCAPE_CHAR, 1) ||
-				    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-					log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-					return 0;
-				}
-				p1_repstr = p2_repstr + 1;
-			}
-
-			if (!dm_pool_grow_object(rh->mem, p1_repstr, 0)) {
-				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-				return 0;
-			}
-		} else {
-			if (!dm_pool_grow_object(rh->mem, repstr, 0)) {
-				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-				return 0;
-			}
-		}
-	} else {
+	if (rh->flags & DM_REPORT_OUTPUT_ALIGNED) {
 		if (!(align = field->props->flags & DM_REPORT_FIELD_ALIGN_MASK))
 			align = ((field->props->flags & DM_REPORT_FIELD_TYPE_NUMBER) ||
-				 (field->props->flags & DM_REPORT_FIELD_TYPE_SIZE)) ? 
+				 (field->props->flags & DM_REPORT_FIELD_TYPE_SIZE)) ?
 				DM_REPORT_FIELD_ALIGN_RIGHT : DM_REPORT_FIELD_ALIGN_LEFT;
 
+		width = field->props->width;
+
 		/* Including trailing '\0'! */
 		buf_size = width + 1;
 		if (!(buf = malloc(buf_size))) {
@@ -4477,7 +4527,7 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 
 		if (align & DM_REPORT_FIELD_ALIGN_LEFT) {
 			if (dm_snprintf(buf, buf_size, "%-*.*s",
-					 width, width, repstr) < 0) {
+					 width, width, field->report_string) < 0) {
 				log_error("dm_report: left-aligned snprintf() failed");
 				goto bad;
 			}
@@ -4487,7 +4537,7 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 			}
 		} else if (align & DM_REPORT_FIELD_ALIGN_RIGHT) {
 			if (dm_snprintf(buf, buf_size, "%*.*s",
-					 width, width, repstr) < 0) {
+					 width, width, field->report_string) < 0) {
 				log_error("dm_report: right-aligned snprintf() failed");
 				goto bad;
 			}
@@ -4496,6 +4546,11 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 				goto bad;
 			}
 		}
+	} else {
+		if (!dm_pool_grow_object(rh->mem, field->report_string, 0)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
 	}
 
 	if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
@@ -4505,21 +4560,164 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 				goto bad;
 			}
 		}
-	} else if (_is_json_report(rh)) {
-		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-			goto bad;
-		}
 	}
 
 	free(buf);
 	return 1;
-
 bad:
 	free(buf);
 	return 0;
 }
 
+static int _safe_repstr_output(struct dm_report *rh, const char *repstr, size_t len)
+{
+	const char *p_repstr;
+	const char *repstr_end = len ? repstr + len : repstr + strlen(repstr);
+
+	/* Escape any JSON_QUOTE that may appear in reported string. */
+	while (1) {
+		if (!(p_repstr = memchr(repstr, JSON_QUOTE[0], repstr_end - repstr)))
+			break;
+
+		if (p_repstr > repstr) {
+			if (!dm_pool_grow_object(rh->mem, repstr, p_repstr - repstr)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+		}
+		if (!dm_pool_grow_object(rh->mem, JSON_ESCAPE_CHAR, 1) ||
+		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+		repstr = p_repstr + 1;
+	}
+
+	if (!dm_pool_grow_object(rh->mem, repstr, repstr_end - repstr)) {
+		log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int _output_field_json_fmt(struct dm_report *rh, struct dm_report_field *field)
+{
+	const char *repstr;
+	size_t list_size, i;
+	struct pos_len *pos_len;
+
+	if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
+	    !dm_pool_grow_object(rh->mem, _get_field_id(rh, field),  0) ||
+	    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
+	    !dm_pool_grow_object(rh->mem, JSON_PAIR, 1)) {
+		log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+		return 0;
+	}
+
+	if (field->props->flags & DM_REPORT_FIELD_TYPE_STRING_LIST) {
+		if (!_is_json_std_report(rh)) {
+
+			/* string list in JSON - report whole list as simple string in quotes */
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			if (!_safe_repstr_output(rh, field->report_string, 0))
+				return_0;
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			return 1;
+		}
+
+		/* string list in JSON_STD - report list as proper JSON array */
+
+		if (!dm_pool_grow_object(rh->mem, JSON_ARRAY_START, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+
+		if (*field->report_string != 0) {
+			pos_len = (struct pos_len *) (field->report_string +
+				  ((struct str_list_sort_value *) field->sort_value)->items[0].len + 1);
+			list_size = pos_len->pos;
+		} else
+			list_size = 0;
+
+		for (i = 0; i < list_size; i++) {
+			pos_len++;
+
+			if (i != 0) {
+				if (!dm_pool_grow_object(rh->mem, JSON_SEPARATOR, 1)) {
+					log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+					return 0;
+				}
+			}
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			if (!_safe_repstr_output(rh, field->report_string + pos_len->pos, pos_len->len))
+				return_0;
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+		}
+
+		if (!dm_pool_grow_object(rh->mem, JSON_ARRAY_END, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+
+		return 1;
+	}
+
+	/* all other types than string list - handle both JSON and JSON_STD */
+
+	if (!(_is_json_std_report(rh) && _is_pure_numeric_field(field))) {
+		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+	}
+
+	if (_is_json_std_report(rh) && _is_pure_numeric_field(field) && !*field->report_string)
+		repstr = JSON_NULL;
+	else
+		repstr = field->report_string;
+
+	if (!_safe_repstr_output(rh, repstr, 0))
+		return_0;
+
+	if (!(_is_json_std_report(rh) && _is_pure_numeric_field(field))) {
+		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
+/*
+ * Produce report output
+ */
+static int _output_field(struct dm_report *rh, struct dm_report_field *field)
+{
+	return _is_json_report(rh) ? _output_field_json_fmt(rh, field)
+				   : _output_field_basic_fmt(rh, field);
+}
+
 static void _destroy_rows(struct dm_report *rh)
 {
 	/*
@@ -4983,6 +5181,7 @@ int dm_report_group_push(struct dm_report_group *group, struct dm_report *report
 				goto_bad;
 			break;
 		case DM_REPORT_GROUP_JSON:
+		case DM_REPORT_GROUP_JSON_STD:
 			if (!_report_group_push_json(item, data))
 				goto_bad;
 			break;
@@ -5046,6 +5245,7 @@ int dm_report_group_pop(struct dm_report_group *group)
 				return_0;
 			break;
 		case DM_REPORT_GROUP_JSON:
+		case DM_REPORT_GROUP_JSON_STD:
 			if (!_report_group_pop_json(item))
 				return_0;
 			break;
@@ -5082,7 +5282,7 @@ int dm_report_group_output_and_pop_all(struct dm_report_group *group)
 			return_0;
 	}
 
-	if (group->type == DM_REPORT_GROUP_JSON) {
+	if (group->type == DM_REPORT_GROUP_JSON || group->type == DM_REPORT_GROUP_JSON_STD) {
 		_json_output_start(group);
 		log_print(JSON_OBJECT_END);
 		group->indent -= JSON_INDENT_UNIT;

device_mapper/vdo/target.h

@@ -77,8 +77,10 @@ enum dm_vdo_write_policy {
 struct dm_vdo_target_params {
 	uint32_t minimum_io_size;       // in sectors
 	uint32_t block_map_cache_size_mb;
-	uint32_t block_map_era_length;	// format period
-
+	union {
+		uint32_t block_map_era_length;	// format period
+		uint32_t block_map_period;      // supported alias
+	};
 	uint32_t check_point_frequency;
 	uint32_t index_memory_size_mb;  // format
 
@@ -106,6 +108,8 @@ struct dm_vdo_target_params {
 bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 				   uint64_t vdo_size);
 
+bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks);
+
 //----------------------------------------------------------------
 
 #endif

device_mapper/vdo/vdo_limits.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2018-2022 Red Hat, Inc. All rights reserved.
  *
  * This file is part of the device-mapper userspace tools.
  *
@@ -15,49 +15,52 @@
 #ifndef DEVICE_MAPPER_VDO_LIMITS_H
 #define DEVICE_MAPPER_VDO_LIMITS_H
 
+#ifndef SECTOR_SHIFT
+#define SECTOR_SHIFT 9L
+#endif
+
 #define DM_VDO_BLOCK_SIZE			UINT64_C(8)		// 4KiB in sectors
+#define DM_VDO_BLOCK_SIZE_KB			(DM_VDO_BLOCK_SIZE << SECTOR_SHIFT)
 
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB	(128)			// 128MiB
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB	(16 * 1024 * 1024 - 1)	// 16TiB - 1
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_PER_LOGICAL_THREAD  (4096 * DM_VDO_BLOCK_SIZE_KB)
 
-#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM	(1)
-#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM	(16380)
+#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM	1
+#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM	16380
 
-#define DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB	(256)			// 0.25 GiB
+#define DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB	256			// 0.25 GiB
 #define DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB	(1024 * 1024 * 1024)	// 1TiB
 
-//#define DM_VDO_READ_CACHE_SIZE_MINIMUM_MB	(0)
-#define DM_VDO_READ_CACHE_SIZE_MAXIMUM_MB	(16 * 1024 * 1024 - 1)	// 16TiB - 1
-
-#define DM_VDO_SLAB_SIZE_MINIMUM_MB		(128)			// 128MiB
+#define DM_VDO_SLAB_SIZE_MINIMUM_MB		128			// 128MiB
 #define DM_VDO_SLAB_SIZE_MAXIMUM_MB		(32 * 1024)		// 32GiB
+#define DM_VDO_SLABS_MAXIMUM			8192
 
-//#define DM_VDO_LOGICAL_SIZE_MINIMUM_MB	(0)
-#define DM_VDO_LOGICAL_SIZE_MAXIMUM_MB	(UINT64_C(4) * 1024 * 1024 * 1024) // 4PiB
+#define DM_VDO_LOGICAL_SIZE_MAXIMUM	(UINT64_C(4) * 1024 * 1024 * 1024 * 1024 * 1024 >> SECTOR_SHIFT) // 4PiB
+#define DM_VDO_PHYSICAL_SIZE_MAXIMUM	(UINT64_C(64) * DM_VDO_BLOCK_SIZE_KB * 1024 * 1024 * 1024 >> SECTOR_SHIFT) // 256TiB
 
-//#define DM_VDO_ACK_THREADS_MINIMUM		(0)
-#define DM_VDO_ACK_THREADS_MAXIMUM		(100)
+#define DM_VDO_ACK_THREADS_MINIMUM		0
+#define DM_VDO_ACK_THREADS_MAXIMUM		100
 
-#define DM_VDO_BIO_THREADS_MINIMUM		(1)
-#define DM_VDO_BIO_THREADS_MAXIMUM		(100)
+#define DM_VDO_BIO_THREADS_MINIMUM		1
+#define DM_VDO_BIO_THREADS_MAXIMUM		100
 
-#define DM_VDO_BIO_ROTATION_MINIMUM		(1)
-#define DM_VDO_BIO_ROTATION_MAXIMUM		(1024)
+#define DM_VDO_BIO_ROTATION_MINIMUM		1
+#define DM_VDO_BIO_ROTATION_MAXIMUM		1024
 
-#define DM_VDO_CPU_THREADS_MINIMUM		(1)
-#define DM_VDO_CPU_THREADS_MAXIMUM		(100)
+#define DM_VDO_CPU_THREADS_MINIMUM		1
+#define DM_VDO_CPU_THREADS_MAXIMUM		100
 
-//#define DM_VDO_HASH_ZONE_THREADS_MINIMUM	(0)
-#define DM_VDO_HASH_ZONE_THREADS_MAXIMUM	(100)
+#define DM_VDO_HASH_ZONE_THREADS_MINIMUM	0
+#define DM_VDO_HASH_ZONE_THREADS_MAXIMUM	100
 
-//#define DM_VDO_LOGICAL_THREADS_MINIMUM	(0)
-#define DM_VDO_LOGICAL_THREADS_MAXIMUM		(100)
+#define DM_VDO_LOGICAL_THREADS_MINIMUM		0
+#define DM_VDO_LOGICAL_THREADS_MAXIMUM		60
 
-//#define DM_VDO_PHYSICAL_THREADS_MINIMUM	(0)
-#define DM_VDO_PHYSICAL_THREADS_MAXIMUM		(16)
+#define DM_VDO_PHYSICAL_THREADS_MINIMUM		0
+#define DM_VDO_PHYSICAL_THREADS_MAXIMUM		16
 
-#define DM_VDO_MAX_DISCARD_MINIMUM		(1)
-#define DM_VDO_MAX_DISCARD_MAXIMUM		(UINT32_MAX / 4096)
+#define DM_VDO_MAX_DISCARD_MINIMUM		1
+#define DM_VDO_MAX_DISCARD_MAXIMUM		(UINT32_MAX / (uint32_t)(DM_VDO_BLOCK_SIZE_KB))
 
 #endif // DEVICE_MAPPER_VDO_LIMITS_H

device_mapper/vdo/vdo_reader.c

@@ -0,0 +1,279 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of the device-mapper userspace tools.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/*
+ * Based on VDO sources: https://github.com/dm-vdo/vdo
+ *
+ * Simplified parser of VDO superblock to obtain basic VDO parameteers
+ *
+ * TODO: maybe switch to some library in the future
+ */
+
+//#define _GNU_SOURCE 1
+//#define _LARGEFILE64_SOURCE 1
+
+#include "device_mapper/misc/dmlib.h"
+
+#include "target.h"
+
+#include "lib/mm/xlate.h"
+//#include "linux/byteorder/big_endian.h"
+//#include "linux/byteorder/little_endian.h"
+//#define le32_to_cpu __le32_to_cpu
+//#define le64_to_cpu __le64_to_cpu
+
+
+#include <errno.h>
+#include <fcntl.h>
+#include <linux/fs.h>	/* For block ioctl definitions */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+typedef unsigned char uuid_t[16];
+
+#define __packed	__attribute__((packed))
+
+static const char _MAGIC_NUMBER[] = "dmvdo001";
+#define MAGIC_NUMBER_SIZE  (sizeof(_MAGIC_NUMBER) - 1)
+
+struct vdo_version_number {
+	uint32_t major_version;
+	uint32_t minor_version;
+} __packed;
+
+/*
+ * The registry of component ids for use in headers
+ */
+enum {
+	SUPER_BLOCK	  = 0,
+	FIXED_LAYOUT	  = 1,
+	RECOVERY_JOURNAL  = 2,
+	SLAB_DEPOT	  = 3,
+	BLOCK_MAP	  = 4,
+	GEOMETRY_BLOCK	  = 5,
+}; /* ComponentID */
+
+struct vdo_header {
+	uint32_t id; /* The component this is a header for */
+	struct vdo_version_number version; /* The version of the data format */
+	size_t size; /* The size of the data following this header */
+} __packed;
+
+struct vdo_geometry_block {
+	char magic_number[MAGIC_NUMBER_SIZE];
+	struct vdo_header header;
+	uint32_t checksum;
+} __packed;
+
+struct vdo_config {
+	uint64_t logical_blocks; /* number of logical blocks */
+	uint64_t physical_blocks; /* number of physical blocks */
+	uint64_t slab_size; /* number of blocks in a slab */
+	uint64_t recovery_journal_size; /* number of recovery journal blocks */
+	uint64_t slab_journal_blocks; /* number of slab journal blocks */
+} __packed;
+
+struct vdo_component_41_0 {
+	uint32_t state;
+	uint64_t complete_recoveries;
+	uint64_t read_only_recoveries;
+	struct vdo_config config; /* packed */
+	uint64_t nonce;
+} __packed;
+
+enum vdo_volume_region_id {
+	VDO_INDEX_REGION = 0,
+	VDO_DATA_REGION = 1,
+	VDO_VOLUME_REGION_COUNT,
+};
+
+struct vdo_volume_region {
+	/* The ID of the region */
+	enum vdo_volume_region_id id;
+	/*
+	 * The absolute starting offset on the device. The region continues
+	 * until the next region begins.
+	 */
+	uint64_t start_block;
+} __packed;
+
+struct vdo_index_config {
+	uint32_t mem;
+	uint32_t unused;
+	uint8_t sparse;
+} __packed;
+
+struct vdo_volume_geometry {
+	uint32_t release_version;
+	uint64_t nonce;
+	uuid_t uuid;
+	uint64_t bio_offset;
+	struct vdo_volume_region regions[VDO_VOLUME_REGION_COUNT];
+	struct vdo_index_config index_config;
+} __packed;
+
+/* Decoding mostly only some used stucture members */
+
+static void _vdo_decode_version(struct vdo_version_number *v)
+{
+	v->major_version = le32_to_cpu(v->major_version);
+	v->minor_version = le32_to_cpu(v->minor_version);
+}
+
+static void _vdo_decode_header(struct vdo_header *h)
+{
+	h->id = le32_to_cpu(h->id);
+	_vdo_decode_version(&h->version);
+	h->size = le64_to_cpu(h->size);
+}
+
+static void _vdo_decode_geometry_region(struct vdo_volume_region *vr)
+{
+	vr->id = le32_to_cpu(vr->id);
+	vr->start_block = le32_to_cpu(vr->start_block);
+}
+
+static void _vdo_decode_volume_geometry(struct vdo_volume_geometry *vg)
+{
+	vg->release_version = le64_to_cpu(vg->release_version);
+	vg->nonce = le64_to_cpu(vg->nonce);
+	_vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]);
+}
+
+static void _vdo_decode_config(struct vdo_config *vc)
+{
+	vc->logical_blocks = le64_to_cpu(vc->logical_blocks);
+	vc->physical_blocks = le64_to_cpu(vc->physical_blocks);
+	vc->slab_size = le64_to_cpu(vc->slab_size);
+	vc->recovery_journal_size = le64_to_cpu(vc->recovery_journal_size);
+	vc->slab_journal_blocks = le64_to_cpu(vc->slab_journal_blocks);
+}
+
+static void _vdo_decode_pvc(struct vdo_component_41_0 *pvc)
+{
+	_vdo_decode_config(&pvc->config);
+	pvc->nonce = le64_to_cpu(pvc->nonce);
+}
+
+bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks)
+{
+	char buffer[4096];
+	int fh, n;
+	bool r = false;
+	off_t l;
+	struct stat st;
+	uint64_t size;
+	uint64_t regpos;
+
+	struct vdo_header h;
+	struct vdo_version_number vn;
+	struct vdo_volume_geometry vg;
+	struct vdo_component_41_0 pvc;
+
+	*logical_blocks = 0;
+	if ((fh = open(vdo_path, O_RDONLY)) == -1) {
+		log_sys_debug("Failed to open VDO backend %s.", vdo_path);
+		return false;
+	}
+
+	if (ioctl(fh, BLKGETSIZE64, &size) == -1) {
+		if (errno != ENOTTY) {
+			log_sys_debug("ioctl", vdo_path);
+			goto err;
+		}
+
+		/* lets retry for file sizes */
+		if (fstat(fh, &st) < 0) {
+			log_sys_debug("fstat", vdo_path);
+			goto err;
+		}
+
+		size = st.st_size;
+	}
+
+	if ((n = read(fh, buffer, sizeof(buffer))) < 0) {
+		log_sys_debug("read", vdo_path);
+		goto err;
+	}
+
+	if (strncmp(buffer, _MAGIC_NUMBER, MAGIC_NUMBER_SIZE)) {
+		log_debug_activation("Found mismatching VDO magic header in %s.", vdo_path);
+		goto err;
+	}
+
+	memcpy(&h, buffer + MAGIC_NUMBER_SIZE, sizeof(h));
+	_vdo_decode_header(&h);
+
+	if  (h.version.major_version != 5) {
+		log_debug_activation("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version);
+		goto err;
+	}
+
+	memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg));
+	_vdo_decode_volume_geometry(&vg);
+
+	regpos = vg.regions[VDO_DATA_REGION].start_block * 4096;
+
+	if ((regpos + sizeof(buffer)) > size) {
+		log_debug_activation("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size);
+		goto err;
+	}
+
+	if ((l = lseek(fh, regpos, SEEK_SET)) < 0) {
+		log_sys_debug("lseek", vdo_path);
+		goto err;
+	}
+
+	if ((n = read(fh, buffer, sizeof(buffer))) < 0) {
+		log_sys_debug("read", vdo_path);
+		goto err;
+	}
+
+
+	memcpy(&vn, buffer + sizeof(struct vdo_geometry_block), sizeof(vn));
+	_vdo_decode_version(&vn);
+
+	if (vn.major_version > 41) {
+		log_debug_activation("Unknown VDO component version %u.", vn.major_version); // should be 41!
+		goto err;
+	}
+
+	memcpy(&pvc, buffer + sizeof(struct vdo_geometry_block) + sizeof(vn), sizeof(pvc));
+	_vdo_decode_pvc(&pvc);
+
+	if (pvc.nonce != vg.nonce) {
+		log_debug_activation("VDO metadata has mismatching VDO nonces " FMTu64 " != " FMTu64 ".", pvc.nonce, vg.nonce);
+		goto err;
+	}
+
+#if 0
+	log_debug_activation("LogBlocks    " FMTu64 ".", pvc.config.logical_blocks);
+	log_debug_activation("PhyBlocks    " FMTu64 ".", pvc.config.physical_blocks);
+	log_debug_activation("SlabSize     " FMTu64 ".", pvc.config.slab_size);
+	log_debug_activation("RecJourSize  " FMTu64 ".", pvc.config.recovery_journal_size);
+	log_debug_activation("SlabJouSize  " FMTu64 ".", pvc.config.slab_journal_blocks);
+#endif
+
+	*logical_blocks = pvc.config.logical_blocks;
+	r = true;
+err:
+	(void) close(fh);
+
+	return r;
+}

device_mapper/vdo/vdo_target.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2018-2022 Red Hat, Inc. All rights reserved.
  *
  * This file is part of LVM2.
  *
@@ -18,82 +18,117 @@
 #include "vdo_limits.h"
 #include "target.h"
 
+/* validate vdo target parameters and  'vdo_size' in sectors */
 bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 				   uint64_t vdo_size)
 {
 	bool valid = true;
 
 	/* 512 or 4096 bytes only ATM */
-	if ((vtp->minimum_io_size != 1) &&
-	    (vtp->minimum_io_size != 8)) {
-		log_error("VDO minimum io size %u is unsupported.",
+	if ((vtp->minimum_io_size != (512 >> SECTOR_SHIFT)) &&
+	    (vtp->minimum_io_size != (4096 >> SECTOR_SHIFT))) {
+		log_error("VDO minimum io size %u is unsupported [512, 4096].",
 			  vtp->minimum_io_size);
 		valid = false;
 	}
 
 	if ((vtp->block_map_cache_size_mb < DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB) ||
 	    (vtp->block_map_cache_size_mb > DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB)) {
-		log_error("VDO block map cache size %u out of range.",
-			  vtp->block_map_cache_size_mb);
+		log_error("VDO block map cache size %u MiB is out of range [%u..%u].",
+			  vtp->block_map_cache_size_mb,
+			  DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB,
+			  DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB);
+		valid = false;
+	}
+
+	if ((vtp->block_map_era_length < DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM) ||
+	    (vtp->block_map_era_length > DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM)) {
+		log_error("VDO block map era length %u is out of range [%u..%u].",
+			  vtp->block_map_era_length,
+			  DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM,
+			  DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->index_memory_size_mb < DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB) ||
 	    (vtp->index_memory_size_mb > DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB)) {
-		log_error("VDO index memory size %u out of range.",
-			  vtp->index_memory_size_mb);
+		log_error("VDO index memory size %u MiB is out of range [%u..%u].",
+			  vtp->index_memory_size_mb,
+			  DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB,
+			  DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB);
 		valid = false;
 	}
 
 	if ((vtp->slab_size_mb < DM_VDO_SLAB_SIZE_MINIMUM_MB) ||
 	    (vtp->slab_size_mb > DM_VDO_SLAB_SIZE_MAXIMUM_MB)) {
-		log_error("VDO slab size %u out of range.",
-			  vtp->slab_size_mb);
+		log_error("VDO slab size %u MiB is out of range [%u..%u].",
+			  vtp->slab_size_mb,
+			  DM_VDO_SLAB_SIZE_MINIMUM_MB,
+			  DM_VDO_SLAB_SIZE_MAXIMUM_MB);
 		valid = false;
 	}
 
 	if ((vtp->max_discard < DM_VDO_MAX_DISCARD_MINIMUM) ||
 	    (vtp->max_discard > DM_VDO_MAX_DISCARD_MAXIMUM)) {
-		log_error("VDO max discard %u out of range.",
-			  vtp->max_discard);
+		log_error("VDO max discard %u is out of range [%u..%u].",
+			  vtp->max_discard,
+			  DM_VDO_MAX_DISCARD_MINIMUM,
+			  DM_VDO_MAX_DISCARD_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->ack_threads > DM_VDO_ACK_THREADS_MAXIMUM) {
-		log_error("VDO ack threads %u out of range.", vtp->ack_threads);
+		log_error("VDO ack threads %u is out of range [0..%u].",
+			  vtp->ack_threads,
+			  DM_VDO_ACK_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->bio_threads < DM_VDO_BIO_THREADS_MINIMUM) ||
 	    (vtp->bio_threads > DM_VDO_BIO_THREADS_MAXIMUM)) {
-		log_error("VDO bio threads %u out of range.", vtp->bio_threads);
+		log_error("VDO bio threads %u is out of range [%u..%u].",
+			  vtp->bio_threads,
+			  DM_VDO_BIO_THREADS_MINIMUM,
+			  DM_VDO_BIO_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->bio_rotation < DM_VDO_BIO_ROTATION_MINIMUM) ||
 	    (vtp->bio_rotation > DM_VDO_BIO_ROTATION_MAXIMUM)) {
-		log_error("VDO bio rotation %u out of range.", vtp->bio_rotation);
+		log_error("VDO bio rotation %u is out of range [%u..%u].",
+			  vtp->bio_rotation,
+			  DM_VDO_BIO_ROTATION_MINIMUM,
+			  DM_VDO_BIO_ROTATION_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->cpu_threads < DM_VDO_CPU_THREADS_MINIMUM) ||
 	    (vtp->cpu_threads > DM_VDO_CPU_THREADS_MAXIMUM)) {
-		log_error("VDO cpu threads %u out of range.", vtp->cpu_threads);
+		log_error("VDO cpu threads %u is out of range [%u..%u].",
+			  vtp->cpu_threads,
+			  DM_VDO_CPU_THREADS_MINIMUM,
+			  DM_VDO_CPU_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->hash_zone_threads > DM_VDO_HASH_ZONE_THREADS_MAXIMUM) {
-		log_error("VDO hash zone threads %u out of range.", vtp->hash_zone_threads);
+		log_error("VDO hash zone threads %u is out of range [0..%u].",
+			  vtp->hash_zone_threads,
+			  DM_VDO_HASH_ZONE_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->logical_threads > DM_VDO_LOGICAL_THREADS_MAXIMUM) {
-		log_error("VDO logical threads %u out of range.", vtp->logical_threads);
+		log_error("VDO logical threads %u is out of range [0..%u].",
+			  vtp->logical_threads,
+			  DM_VDO_LOGICAL_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->physical_threads > DM_VDO_PHYSICAL_THREADS_MAXIMUM) {
-		log_error("VDO physical threads %u out of range.", vtp->physical_threads);
+		log_error("VDO physical threads %u is out of range [0..%u].",
+			  vtp->physical_threads,
+			  DM_VDO_PHYSICAL_THREADS_MAXIMUM);
 		valid = false;
 	}
 
@@ -120,10 +155,10 @@ bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 		valid = false;
 	}
 
-	if (vdo_size >= (DM_VDO_LOGICAL_SIZE_MAXIMUM_MB * UINT64_C(1024 * 2))) {
-		log_error("VDO logical size is by " FMTu64 "KiB bigger then limit " FMTu64 "TiB.",
-			  (vdo_size - (DM_VDO_LOGICAL_SIZE_MAXIMUM_MB * UINT64_C(1024 * 2))) / 2,
-			  DM_VDO_LOGICAL_SIZE_MAXIMUM_MB / UINT64_C(1024) / UINT64_C(1024));
+	if (vdo_size > DM_VDO_LOGICAL_SIZE_MAXIMUM) {
+		log_error("VDO logical size is larger than limit " FMTu64 " TiB by " FMTu64 " KiB.",
+			  DM_VDO_LOGICAL_SIZE_MAXIMUM / (UINT64_C(1024) * 1024 * 1024 * 1024 >> SECTOR_SHIFT),
+			  (vdo_size - DM_VDO_LOGICAL_SIZE_MAXIMUM) / 2);
 		valid = false;
 	}
 

doc/kernel/cache-policies.txt

@@ -67,7 +67,7 @@ the entries (each hotspot block covers a larger area than a single
 cache block).
 
 All this means smq uses ~25bytes per cache block.  Still a lot of
-memory, but a substantial improvement nontheless.
+memory, but a substantial improvement nonetheless.
 
 Level balancing:
 mq placed entries in different levels of the multiqueue structures

doc/kernel/crypt.txt

@@ -35,7 +35,7 @@ Parameters: <cipher> <key> <iv_offset> <device path> \
         capi:authenc(hmac(sha256),xts(aes))-random
         capi:rfc7539(chacha20,poly1305)-random
 
-    The /proc/crypto contains a list of curently loaded crypto modes.
+    The /proc/crypto contains a list of currently loaded crypto modes.
 
 <key>
     Key used for encryption. It is encoded either as a hexadecimal number
@@ -81,7 +81,7 @@ Parameters: <cipher> <key> <iv_offset> <device path> \
 
 <#opt_params>
     Number of optional parameters. If there are no optional parameters,
-    the optional paramaters section can be skipped or #opt_params can be zero.
+    the optional parameters section can be skipped or #opt_params can be zero.
     Otherwise #opt_params is the number of following arguments.
 
     Example of optional parameters section:

doc/kernel/integrity.txt

@@ -120,7 +120,7 @@ journal_crypt:algorithm(:key)	(the key is optional)
 	"salsa20", "ctr(aes)" or "ecb(arc4)").
 
 	The journal contains history of last writes to the block device,
-	an attacker reading the journal could see the last sector nubmers
+	an attacker reading the journal could see the last sector numbers
 	that were written. From the sector numbers, the attacker can infer
 	the size of files that were written. To protect against this
 	situation, you can encrypt the journal.

doc/kernel/verity.txt

@@ -65,7 +65,7 @@ Construction Parameters
 
 <#opt_params>
     Number of optional parameters. If there are no optional parameters,
-    the optional paramaters section can be skipped or #opt_params can be zero.
+    the optional parameters section can be skipped or #opt_params can be zero.
     Otherwise #opt_params is the number of following arguments.
 
     Example of optional parameters section:

doc/lvm2-raid.txt

@@ -37,7 +37,7 @@ segment type.  The available RAID types are:
 	"raid6_nr" - RAID6 Rotating parity N with data restart
 	"raid6_nc" - RAID6 Rotating parity N with data continuation
 The exception to 'no shorthand options' will be where the RAID implementations
-can displace traditional tagets.  This is the case with 'mirror' and 'raid1'.
+can displace traditional targets.  This is the case with 'mirror' and 'raid1'.
 In this case, "mirror_segtype_default" - found under the "global" section in
 lvm.conf - can be set to "mirror" or "raid1".  The segment type inferred when
 the '-m' option is used will be taken from this setting.  The default segment
@@ -104,7 +104,7 @@ and 4 devices for RAID 6/10.
 
 lvconvert should work exactly as it does now when dealing with mirrors -
 even if(when) we switch to MD RAID1.  Of course, there are no plans to
-allow the presense of the metadata area to be configurable (e.g. --corelog).
+allow the presence of the metadata area to be configurable (e.g. --corelog).
 It will be simple enough to detect if the LV being up/down-converted is
 new or old-style mirroring.
 
@@ -120,7 +120,7 @@ RAID4 to RAID5 or RAID5 to RAID6.
 Line 02/03/04:
 These are familiar options - all of which would now be available as options
 for change.  (However, it'd be nice if we didn't have regionsize in there.
-It's simple on the kernel side, but is just an extra - often unecessary -
+It's simple on the kernel side, but is just an extra - often unnecessary -
 parameter to many functions in the LVM codebase.)
 
 Line 05:
@@ -375,8 +375,8 @@ the slot.  Even the names of the images will be renamed to properly reflect
 their index in the array.  Unlike the "mirror" segment type, you will never have
 an image named "*_rimage_1" occupying the index position 0.
 
-As with adding images, removing images holds off on commiting LVM metadata
-until all possible changes have been made.  This reduces the likelyhood of bad
+As with adding images, removing images holds off on committing LVM metadata
+until all possible changes have been made.  This reduces the likelihood of bad
 intermediate stages being left due to a failure of operation or machine crash.
 
 RAID1 '--splitmirrors', '--trackchanges', and '--merge' operations

doc/lvm_fault_handling.txt

@@ -87,7 +87,7 @@ are as follows:
   /etc/lvm/lvm.conf.  Once this operation is complete, the logical volumes
   will be consistent.  However, the volume group will still be inconsistent -
   due to the refernced-but-missing device/PV - and operations will still be
-  restricted to the aformentioned actions until either the device is
+  restricted to the aforementioned actions until either the device is
   restored or 'vgreduce --removemissing' is run.
 
 Device Revival (transient failures):
@@ -135,9 +135,9 @@ If a mirror is not 'in-sync', a read failure will produce an I/O error.
 This error will propagate all the way up to the applications above the
 logical volume (e.g. the file system).  No automatic intervention will
 take place in this case either.  It is up to the user to decide what
-can be done/salvaged in this senario.  If the user is confident that the
+can be done/salvaged in this scenario.  If the user is confident that the
 images of the mirror are the same (or they are willing to simply attempt
-to retreive whatever data they can), 'lvconvert' can be used to eliminate
+to retrieve whatever data they can), 'lvconvert' can be used to eliminate
 the failed image and proceed.
 
 Mirror resynchronization errors:
@@ -191,11 +191,11 @@ command are set in the LVM configuration file.  They are:
   3-way mirror fails, the mirror will be converted to a 2-way mirror.
   The "allocate" policy takes the further action of trying to replace
   the failed image using space that is available in the volume group.
-  Replacing a failed mirror image will incure the cost of
+  Replacing a failed mirror image will incur the cost of
   resynchronizing - degrading the performance of the mirror.  The
   default policy for handling an image failure is "remove".  This
   allows the mirror to still function, but gives the administrator the
-  choice of when to incure the extra performance costs of replacing
+  choice of when to incur the extra performance costs of replacing
   the failed image.
 
 RAID logical volume device failures are handled differently from the "mirror"

doc/lvmpolld_overview.txt

@@ -63,7 +63,7 @@ classical snapshot merge, thin snapshot merge.
 
 The second store is suited only for pvmove --abort operations in-progress. Both
 stores are independent and identical LVs (pvmove /dev/sda3 and pvmove --abort /dev/sda3)
-can be run concurently from lvmpolld point of view (on lvm2 side the consistency is
+can be run concurrently from lvmpolld point of view (on lvm2 side the consistency is
 guaranteed by lvm2 locking mechanism).
 
 Locking order

doc/tagging.txt

@@ -126,7 +126,7 @@ Usage Examples
   followed by 'vgchange -ay vg2'
   
   
-  Option (ii) - localised admin & configuation
+  Option (ii) - localised admin & configuration
   (i.e. each host holds *locally* which classes of volumes to activate)
     # Add @database tag to vg1's metadata
     vgchange --addtag @database vg1

doc/udev_assembly.txt

@@ -35,7 +35,7 @@ VGs from PVs as they appear, and at the same time collect information on what is
 already available. A command, pvscan --cache is expected to be used to
 implement udev rules. It is relatively easy to make this command print out a
 list of VGs (and possibly LVs) that have been made available by adding any
-particular device to the set of visible devices. In othe words, udev says "hey,
+particular device to the set of visible devices. In other words, udev says "hey,
 /dev/sdb just appeared", calls pvscan --cache, which talks to lvmetad, which
 says "cool, that makes vg0 complete". Pvscan takes this info and prints it out,
 and the udev rule can then somehow decide whether anything needs to be done

include/Makefile.in

@@ -1,6 +1,6 @@
 #
 # Copyright (C) 2001-2004 Sistina Software, Inc. All rights reserved.
-# Copyright (C) 2004-2018 Red Hat, Inc. All rights reserved.
+# Copyright (C) 2004-2023 Red Hat, Inc. All rights reserved.
 #
 # This file is part of LVM2.
 #
@@ -18,8 +18,23 @@ top_builddir = @top_builddir@
 
 include $(top_builddir)/make.tmpl
 
+.DELETE_ON_ERROR:
+cmds.h: $(top_srcdir)/tools/command-lines.in $(top_srcdir)/tools/license.inc Makefile
+	@echo "    [GEN] $@"
+	$(Q) \
+	( cat $(top_srcdir)/tools/license.inc && \
+	  echo "/* Do not edit. This file is generated by the Makefile. */" && \
+	  echo "cmd(CMD_NONE, none)" && \
+	  trap "$(RM) $@-t" EXIT INT QUIT TERM && \
+	  $(AWK) '/^ID:/ {print "cmd(" $$2 "_CMD, " $$2 ")"}' $< >$@-t && \
+	  LC_ALL=C $(SORT) -u $@-t && \
+	  echo "cmd(CMD_COUNT, count)" \
+	) > $@
+
+all: cmds.h
+
 DISTCLEAN_TARGETS += configure.h lvm-version.h
-CLEAN_TARGETS += \
+CLEAN_TARGETS += cmds.h cmds.h-t \
  .symlinks \
  .symlinks_created \
  activate.h \
@@ -71,6 +86,8 @@ CLEAN_TARGETS += \
  lvm2app.h \
  lvm2cmd.h \
  lvmcache.h \
+ lvmetad-client.h \
+ lvmetad.h \
  lvmlockd-client.h \
  lvmlockd.h \
  lvmnotify.h \
@@ -99,4 +116,5 @@ CLEAN_TARGETS += \
  util.h \
  uuid.h \
  vg.h \
- xlate.h
+ xlate.h \
+ cmds.h

include/configure.h.in

@@ -25,18 +25,13 @@
 /* The path to 'cache_restore', if available. */
 #undef CACHE_RESTORE_CMD
 
-/* Define to 1 if the `closedir' function returns void instead of `int'. */
+/* Define to 1 if the `closedir' function returns void instead of int. */
 #undef CLOSEDIR_VOID
 
 /* Path to cmirrord pidfile. */
 #undef CMIRRORD_PIDFILE
 
-/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
-   systems. This function is required for `alloca.c' support on those systems.
-   */
-#undef CRAY_STACKSEG_END
-
-/* Define to 1 if using `alloca.c'. */
+/* Define to 1 if using 'alloca.c'. */
 #undef C_ALLOCA
 
 /* Name of default metadata archive subdirectory. */
@@ -90,7 +85,7 @@
 /* Use blkid wiping by default. */
 #undef DEFAULT_USE_BLKID_WIPING
 
-/* Default for lvm.conf use_devicefile. */
+/* Default for lvm.conf use_devicesfile. */
 #undef DEFAULT_USE_DEVICES_FILE
 
 /* Use lvmlockd by default. */
@@ -114,9 +109,6 @@
 /* Define to 1 to enable the device-mapper filemap daemon. */
 #undef DMFILEMAPD
 
-/* Define to enable compat protocol */
-#undef DM_COMPAT
-
 /* Define default group for device node */
 #undef DM_DEVICE_GID
 
@@ -144,11 +136,10 @@
 /* Define to 1 if you have the `alarm' function. */
 #undef HAVE_ALARM
 
-/* Define to 1 if you have `alloca', as a function or macro. */
+/* Define to 1 if you have 'alloca', as a function or macro. */
 #undef HAVE_ALLOCA
 
-/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix).
-   */
+/* Define to 1 if <alloca.h> works. */
 #undef HAVE_ALLOCA_H
 
 /* Define to 1 if you have the <arpa/inet.h> header file. */
@@ -163,6 +154,9 @@
 /* Define to 1 if you have the `atexit' function. */
 #undef HAVE_ATEXIT
 
+/* Define if blkid.h has BLKID_SUBLKS_FSINFO */
+#undef HAVE_BLKID_SUBLKS_FSINFO
+
 /* Define if ioctl BLKZEROOUT can be used for device zeroing. */
 #undef HAVE_BLKZEROOUT
 
@@ -276,6 +270,9 @@
 /* Define to 1 if you have the <machine/endian.h> header file. */
 #undef HAVE_MACHINE_ENDIAN_H
 
+/* Define to 1 if you have the `mallinfo2' function. */
+#undef HAVE_MALLINFO2
+
 /* Define to 1 if your system has a GNU libc compatible `malloc' function, and
    to 0 otherwise. */
 #undef HAVE_MALLOC
@@ -286,9 +283,6 @@
 /* Define to 1 if you have the `memchr' function. */
 #undef HAVE_MEMCHR
 
-/* Define to 1 if you have the <memory.h> header file. */
-#undef HAVE_MEMORY_H
-
 /* Define to 1 if you have the `memset' function. */
 #undef HAVE_MEMSET
 
@@ -405,7 +399,7 @@
 /* Define to 1 if you have the `strerror' function. */
 #undef HAVE_STRERROR
 
-/* Define to 1 if you have the `strerror_r' function. */
+/* Define if you have `strerror_r'. */
 #undef HAVE_STRERROR_R
 
 /* Define to 1 if you have the <strings.h> header file. */
@@ -607,6 +601,9 @@
 /* Path to lvm binary. */
 #undef LVM_PATH
 
+/* Path to lvresize_fs_helper script. */
+#undef LVRESIZE_FS_HELPER_PATH
+
 /* Define to 1 if `major', `minor', and `makedev' are declared in <mkdev.h>.
    */
 #undef MAJOR_IN_MKDEV
@@ -651,9 +648,6 @@
 /* Define to 1 to include the LVM readline shell. */
 #undef READLINE_SUPPORT
 
-/* Define as the return type of signal handlers (`int' or `void'). */
-#undef RETSIGTYPE
-
 /* Define to 1 to include built-in support for snapshots. */
 #undef SNAPSHOT_INTERNAL
 
@@ -665,7 +659,9 @@
 	STACK_DIRECTION = 0 => direction of growth unknown */
 #undef STACK_DIRECTION
 
-/* Define to 1 if you have the ANSI C header files. */
+/* Define to 1 if all of the C90 standard headers exist (not just the ones
+   required in a freestanding environment). This macro is provided for
+   backward compatibility; new code need not use it. */
 #undef STDC_HEADERS
 
 /* Define to 1 if strerror_r returns char *. */
@@ -696,9 +692,6 @@
 /* The path to 'thin_restore', if available. */
 #undef THIN_RESTORE_CMD
 
-/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
-#undef TIME_WITH_SYS_TIME
-
 /* Define to 1 if your <sys/time.h> declares `struct tm'. */
 #undef TM_IN_SYS_TIME
 
@@ -775,7 +768,7 @@
 /* Define to `long int' if <sys/types.h> does not define. */
 #undef off_t
 
-/* Define to `int' if <sys/types.h> does not define. */
+/* Define as a signed integer type capable of holding a process identifier. */
 #undef pid_t
 
 /* Define to rpl_realloc if the replacement function should be used. */

lib/Makefile.in

@@ -40,6 +40,10 @@ SOURCES =\
 	device/dev-luks.c \
 	device/dev-dasd.c \
 	device/dev-lvm1-pool.c \
+	device/filesystem.c \
+	device/online.c \
+	device/parse_vpd.c \
+	device/dev_util.c \
 	display/display.c \
 	error/errseg.c \
 	unknown/unknown.c \
@@ -53,7 +57,6 @@ SOURCES =\
 	filters/filter-partitioned.c \
 	filters/filter-type.c \
 	filters/filter-usable.c \
-	filters/filter-internal.c \
 	filters/filter-signature.c \
 	filters/filter-deviceid.c \
 	format_text/archive.c \

lib/activate/activate.c

@@ -322,6 +322,11 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent)
 {
 	return 0;
 }
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg)
+{
+	return 0;
+}
 int lvs_in_vg_activated(const struct volume_group *vg)
 {
 	return 0;
@@ -413,7 +418,7 @@ int add_areas_line(struct dev_manager *dm, struct lv_segment *seg,
 {
         return 0;
 }
-int device_is_usable(struct device *dev, struct dev_usable_check_params check, int *is_lv)
+int device_is_usable(struct cmd_context *cmd, struct device *dev, struct dev_usable_check_params check, int *is_lv)
 {
         return 0;
 }
@@ -486,12 +491,20 @@ int library_version(char *version, size_t size)
 
 int driver_version(char *version, size_t size)
 {
+	static char _vsn[80] = { 0 };
+
 	if (!activation())
 		return 0;
 
 	log_very_verbose("Getting driver version");
 
-	return dm_driver_version(version, size);
+	if (!_vsn[0] &&
+	    !dm_driver_version(_vsn, sizeof(_vsn)))
+		return_0;
+
+	(void) dm_strncpy(version, _vsn, size);
+
+	return 1;
 }
 
 int target_version(const char *target_name, uint32_t *maj,
@@ -616,6 +629,15 @@ int target_present(struct cmd_context *cmd, const char *target_name,
 				      &maj, &min, &patchlevel);
 }
 
+int get_device_list(const struct volume_group *vg, struct dm_list **devs,
+		    unsigned *devs_features)
+{
+	if (!activation())
+		return 0;
+
+	return dev_manager_get_device_list(NULL, devs, devs_features);
+}
+
 /*
  * When '*info' is NULL, returns 1 only when LV is active.
  * When '*info' != NULL, returns 1 when info structure is populated.
@@ -1346,6 +1368,32 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent)
 	return 1;
 }
 
+/*
+ * lv_vdo_pool_size_config obtains size configuration from active VDO table line
+ *
+ * If the 'params' string has been already retrieved, use it.
+ * If the mempool already exists, use it.
+ *
+ */
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg)
+{
+	struct dev_manager *dm;
+	int r;
+
+	if (!lv_info(lv->vg->cmd, lv, 1, NULL, 0, 0))
+		return 1;  /* Inactive VDO pool -> no runtime config */
+
+	if (!(dm = dev_manager_create(lv->vg->cmd, lv->vg->name, !lv_is_pvmove(lv))))
+		return_0;
+
+	r = dev_manager_vdo_pool_size_config(dm, lv, cfg);
+
+	dev_manager_destroy(dm);
+
+	return r;
+}
+
 static int _lv_active(struct cmd_context *cmd, const struct logical_volume *lv)
 {
 	struct lvinfo info;
@@ -2124,7 +2172,11 @@ static int _lv_suspend(struct cmd_context *cmd, const char *lvid_s,
 	 * TODO: Relax this limiting condition further */
 	if (!flush_required &&
 	    (lv_is_pvmove(lv) || pvmove_lv ||
-	     (!lv_is_mirror(lv) && !lv_is_thin_pool(lv) && !lv_is_thin_volume(lv)))) {
+	     (!lv_is_mirror(lv) &&
+	      !lv_is_thin_volume(lv) &&
+	      !lv_is_thin_pool(lv) &&
+	      !lv_is_vdo(lv) &&
+	      !lv_is_vdo_pool(lv)))) {
 		log_debug("Requiring flush for LV %s.", display_lvname(lv));
 		flush_required = 1;
 	}
@@ -2374,6 +2426,7 @@ int lv_deactivate(struct cmd_context *cmd, const char *lvid_s, const struct logi
 	static const struct lv_activate_opts laopts = { .skip_in_use = 1 };
 	struct dm_list *snh;
 	int r = 0;
+	unsigned tmp_state;
 
 	if (!activation())
 		return 1;
@@ -2446,12 +2499,17 @@ int lv_deactivate(struct cmd_context *cmd, const char *lvid_s, const struct logi
 	}
 	critical_section_dec(cmd, "deactivated");
 
+	tmp_state = cmd->disable_dm_devs;
+	cmd->disable_dm_devs = 1;
+
 	if (!lv_info(cmd, lv, 0, &info, 0, 0) || info.exists) {
 		/* Turn into log_error, but we do not log error */
 		log_debug_activation("Deactivated volume is still %s present.",
 				     display_lvname(lv));
 		r = 0;
 	}
+
+	cmd->disable_dm_devs = tmp_state;
 out:
 
 	return r;
@@ -2737,7 +2795,7 @@ static int _component_cb(struct logical_volume *lv, void *data)
 
 	if (lv_is_locked(lv) || lv_is_pvmove(lv) ||/* ignoring */
 	    /* thin-pool is special and it's using layered device */
-	    (lv_is_thin_pool(lv) && pool_is_active(lv)))
+	    (lv_is_thin_pool(lv) && thin_pool_is_active(lv)))
 		return -1;
 
 	/* External origin is activated through thinLV and uses -real suffix.

lib/activate/activate.h

@@ -106,6 +106,10 @@ int target_present(struct cmd_context *cmd, const char *target_name,
 		   int use_modprobe);
 int target_version(const char *target_name, uint32_t *maj,
 		   uint32_t *min, uint32_t *patchlevel);
+
+int get_device_list(const struct volume_group *vg, struct dm_list **devs,
+		    unsigned *devs_features);
+
 int raid4_is_supported(struct cmd_context *cmd, const struct segment_type *segtype);
 int lvm_dm_prefix_check(int major, int minor, const char *prefix);
 int list_segment_modules(struct dm_pool *mem, const struct lv_segment *seg,
@@ -200,6 +204,8 @@ int lv_thin_pool_status(const struct logical_volume *lv, int flush,
 int lv_vdo_pool_status(const struct logical_volume *lv, int flush,
 		       struct lv_status_vdo **status);
 int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent);
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg);
 
 /*
  * Return number of LVs in the VG that are active.
@@ -254,7 +260,7 @@ struct dev_usable_check_params {
  * Returns 1 if mapped device is not suspended, blocked or
  * is using a reserved name.
  */
-int device_is_usable(struct device *dev, struct dev_usable_check_params check, int *is_lv);
+int device_is_usable(struct cmd_context *cmd, struct device *dev, struct dev_usable_check_params check, int *is_lv);
 
 /*
  * Declaration moved here from fs.h to keep header fs.h hidden

lib/activate/dev_manager.c

@@ -107,6 +107,8 @@ static struct dm_task *_setup_task_run(int task, struct dm_info *info,
 				       int with_flush,
 				       int query_inactive)
 {
+	char vsn[80];
+	unsigned maj, min;
 	struct dm_task *dmt;
 
 	if (!(dmt = dm_task_create(task)))
@@ -138,8 +140,20 @@ static struct dm_task *_setup_task_run(int task, struct dm_info *info,
 	if (!with_flush && !dm_task_no_flush(dmt))
 		log_warn("WARNING: Failed to set no_flush.");
 
-	if (task == DM_DEVICE_TARGET_MSG)
+	switch (task) {
+	case DM_DEVICE_TARGET_MSG:
 		return dmt; /* TARGET_MSG needs more local tweaking before task_run() */
+	case DM_DEVICE_LIST:
+		/* Use 'newuuid' only with DM version that supports it */
+		if (driver_version(vsn, sizeof(vsn)) &&
+		    (sscanf(vsn, "%u.%u", &maj, &min) == 2) &&
+		    (maj == 4 ? min >= 19 : maj > 4) &&
+		    !dm_task_set_newuuid(dmt, " ")) // new uuid has no meaning here
+			log_warn("WARNING: Failed to query uuid with LIST.");
+		break;
+	default:
+		break;
+	}
 
 	if (!dm_task_run(dmt))
 		goto_out;
@@ -190,7 +204,7 @@ static int _get_segment_status_from_target_params(const char *target_name,
 	    /* If kernel's type isn't an exact match is it compatible? */
 	    (!segtype->ops->target_status_compatible ||
 	     !segtype->ops->target_status_compatible(target_name))) {
-		log_warn(INTERNAL_ERROR "WARNING: Segment type %s found does not match expected type %s for %s.",
+		log_warn("WARNING: Detected %s segment type does not match expected type %s for %s.",
 			 target_name, segtype->name, display_lvname(seg_status->seg->lv));
 		return 0;
 	}
@@ -263,7 +277,7 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 	int dmtask;
 	int with_flush; /* TODO: arg for _info_run */
 	void *target = NULL;
-	uint64_t target_start, target_length, start, length, length_crop = 0;
+	uint64_t target_start, target_length, start, extent_size, length, length_crop = 0;
 	char *target_name, *target_params;
 	const char *devname;
 
@@ -292,8 +306,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 
 	/* Query status only for active device */
 	if (seg_status && dminfo->exists) {
-		start = length = seg_status->seg->lv->vg->extent_size;
-		start *= seg_status->seg->le;
+		extent_size = length = seg_status->seg->lv->vg->extent_size;
+		start = extent_size * seg_status->seg->le;
 		length *= _seg_len(seg_status->seg);
 
 		/* Uses max DM_THIN_MAX_METADATA_SIZE sectors for metadata device */
@@ -314,6 +328,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 
 			if ((start == target_start) &&
 			    ((length == target_length) ||
+			     ((lv_is_vdo_pool(seg_status->seg->lv)) && /* should fit within extent size */
+			      (length < target_length) && ((length + extent_size) > target_length)) ||
 			     (length_crop && (length_crop == target_length))))
 				break; /* Keep target_params when matching segment is found */
 
@@ -359,7 +375,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
  *
  * Returns: 1 if mirror should be ignored, 0 if safe to use
  */
-static int _ignore_blocked_mirror_devices(struct device *dev,
+static int _ignore_blocked_mirror_devices(struct cmd_context *cmd,
+					  struct device *dev,
 					  uint64_t start, uint64_t length,
 					  char *mirror_status_str)
 {
@@ -402,7 +419,7 @@ static int _ignore_blocked_mirror_devices(struct device *dev,
 				goto_out;
 
 			tmp_dev->dev = MKDEV(sm->logs[0].major, sm->logs[0].minor);
-			if (device_is_usable(tmp_dev, (struct dev_usable_check_params)
+			if (device_is_usable(cmd, tmp_dev, (struct dev_usable_check_params)
 					     { .check_empty = 1,
 					       .check_blocked = 1,
 					       .check_suspended = ignore_suspended_devices(),
@@ -606,6 +623,60 @@ static int _ignore_frozen_raid(struct device *dev, const char *params)
 	return r;
 }
 
+static int _is_usable_uuid(const struct device *dev, const char *name, const char *uuid, int check_reserved, int check_lv, int *is_lv)
+{
+	char *vgname, *lvname, *layer;
+	char vg_name[NAME_LEN];
+
+	if (!check_reserved && !check_lv)
+		return 1;
+
+	if (!strncmp(uuid, UUID_PREFIX, sizeof(UUID_PREFIX) - 1)) { /* with LVM- prefix */
+		if (check_reserved) {
+			/* Check internal lvm devices */
+			if (strlen(uuid) > (sizeof(UUID_PREFIX) + 2 * ID_LEN)) { /* 68 with suffix */
+				log_debug_activation("%s: Reserved uuid %s on internal LV device %s not usable.",
+						     dev_name(dev), uuid, name);
+				return 0;
+			}
+
+			/* Recognize some older reserved LVs just from the LV name (snapshot, pvmove...) */
+			vgname = vg_name;
+			if (!dm_strncpy(vg_name, name, sizeof(vg_name)) ||
+			    !dm_split_lvm_name(NULL, NULL, &vgname, &lvname, &layer))
+				return_0;
+
+			/* FIXME: fails to handle dev aliases i.e. /dev/dm-5, replace with UUID suffix */
+			if (lvname && (is_reserved_lvname(lvname) || *layer)) {
+				log_debug_activation("%s: Reserved internal LV device %s/%s%s%s not usable.",
+						     dev_name(dev), vgname, lvname, *layer ? "-" : "", layer);
+				return 0;
+			}
+		}
+
+		if (check_lv) {
+			/* Skip LVs */
+			if (is_lv)
+				*is_lv = 1;
+			return 0;
+		}
+	}
+
+	if (check_reserved &&
+	    (!strncmp(uuid, CRYPT_TEMP, sizeof(CRYPT_TEMP) - 1) ||
+	     !strncmp(uuid, CRYPT_SUBDEV, sizeof(CRYPT_SUBDEV) - 1) ||
+	     !strncmp(uuid, STRATIS, sizeof(STRATIS) - 1))) {
+		/* Skip private crypto devices */
+		log_debug_activation("%s: Reserved uuid %s on %s device %s not usable.",
+				     dev_name(dev), uuid,
+				     uuid[0] == 'C' ? "crypto" : "stratis",
+				     name);
+		return 0;
+	}
+
+        return 1;
+}
+
 /*
  * device_is_usable
  * @dev
@@ -622,15 +693,14 @@ static int _ignore_frozen_raid(struct device *dev, const char *params)
  *
  * Returns: 1 if usable, 0 otherwise
  */
-int device_is_usable(struct device *dev, struct dev_usable_check_params check, int *is_lv)
+int device_is_usable(struct cmd_context *cmd, struct device *dev, struct dev_usable_check_params check, int *is_lv)
 {
 	struct dm_task *dmt;
 	struct dm_info info;
 	const char *name, *uuid;
 	uint64_t start, length;
 	char *target_type = NULL;
-	char *params, *vgname, *lvname, *layer;
-	char vg_name[NAME_LEN];
+	char *params;
 	void *next = NULL;
 	int only_error_or_zero_target = 1;
 	int r = 0;
@@ -655,50 +725,9 @@ int device_is_usable(struct device *dev, struct dev_usable_check_params check, i
 		goto out;
 	}
 
-	if (uuid && (check.check_reserved || check.check_lv)) {
-		if (!strncmp(uuid, UUID_PREFIX, sizeof(UUID_PREFIX) - 1)) { /* with LVM- prefix */
-			if (check.check_reserved) {
-				/* Check internal lvm devices */
-				if (strlen(uuid) > (sizeof(UUID_PREFIX) + 2 * ID_LEN)) { /* 68 with suffix */
-					log_debug_activation("%s: Reserved uuid %s on internal LV device %s not usable.",
-							     dev_name(dev), uuid, name);
-					goto out;
-				}
-
-				/* Recognize some older reserved LVs just from the LV name (snapshot, pvmove...) */
-				vgname = vg_name;
-				if (!dm_strncpy(vg_name, name, sizeof(vg_name)) ||
-				    !dm_split_lvm_name(NULL, NULL, &vgname, &lvname, &layer))
-					goto_out;
-
-				/* FIXME: fails to handle dev aliases i.e. /dev/dm-5, replace with UUID suffix */
-				if (lvname && (is_reserved_lvname(lvname) || *layer)) {
-					log_debug_activation("%s: Reserved internal LV device %s/%s%s%s not usable.",
-							     dev_name(dev), vgname, lvname, *layer ? "-" : "", layer);
-					goto out;
-				}
-			}
-
-			if (check.check_lv) {
-				/* Skip LVs */
-				if (is_lv)
-					*is_lv = 1;
-				goto out;
-			}
-		}
-
-		if (check.check_reserved &&
-		    (!strncmp(uuid, CRYPT_TEMP, sizeof(CRYPT_TEMP) - 1) ||
-		     !strncmp(uuid, CRYPT_SUBDEV, sizeof(CRYPT_SUBDEV) - 1) ||
-		     !strncmp(uuid, STRATIS, sizeof(STRATIS) - 1))) {
-			/* Skip private crypto devices */
-			log_debug_activation("%s: Reserved uuid %s on %s device %s not usable.",
-					     dev_name(dev), uuid,
-					     uuid[0] == 'C' ? "crypto" : "stratis",
-					     name);
-			goto out;
-		}
-	}
+	if (uuid &&
+	    !_is_usable_uuid(dev, name, uuid, check.check_reserved, check.check_lv, is_lv))
+		goto out;
 
 	/* FIXME Also check for mpath no paths */
 	do {
@@ -713,7 +742,7 @@ int device_is_usable(struct device *dev, struct dev_usable_check_params check, i
 				log_debug_activation("%s: Scanning mirror devices is disabled.", dev_name(dev));
 				goto out;
 			}
-			if (!_ignore_blocked_mirror_devices(dev, start,
+			if (!_ignore_blocked_mirror_devices(cmd, dev, start,
 							    length, params)) {
 				log_debug_activation("%s: Mirror device %s not usable.",
 						     dev_name(dev), name);
@@ -909,6 +938,25 @@ int dev_manager_check_prefix_dm_major_minor(uint32_t major, uint32_t minor, cons
 	return r;
 }
 
+int dev_manager_get_device_list(const char *prefix, struct dm_list **devs, unsigned *devs_features)
+{
+	struct dm_task *dmt;
+	int r = 1;
+
+	if (!(dmt = _setup_task_run(DM_DEVICE_LIST, NULL, NULL, NULL, 0, 0, 0, 0, 0, 0)))
+		return_0;
+
+	if (!dm_task_get_device_list(dmt, devs, devs_features)) {
+		r = 0;
+		goto_out;
+	}
+
+    out:
+	dm_task_destroy(dmt);
+
+	return r;
+}
+
 int dev_manager_info(struct cmd_context *cmd,
 		     const struct logical_volume *lv, const char *layer,
 		     int with_open_count, int with_read_ahead, int with_name_check,
@@ -924,6 +972,16 @@ int dev_manager_info(struct cmd_context *cmd,
 	if (!(dlid = build_dm_uuid(cmd->mem, lv, layer)))
 		goto_out;
 
+	if (!cmd->disable_dm_devs &&
+	    cmd->cache_dm_devs &&
+	    !dm_device_list_find_by_uuid(cmd->cache_dm_devs, dlid, NULL)) {
+		log_debug("Cached as inactive %s.", name);
+		if (dminfo)
+			memset(dminfo, 0, sizeof(*dminfo));
+		r = 1;
+		goto out;
+	}
+
 	if (!(r = _info(cmd, name, dlid,
 			with_open_count, with_read_ahead, with_name_check,
 			dminfo, read_ahead, seg_status)))
@@ -1899,6 +1957,71 @@ out:
 	return r;
 }
 
+int dev_manager_vdo_pool_size_config(struct dev_manager *dm,
+				     const struct logical_volume *lv,
+				     struct vdo_pool_size_config *cfg)
+{
+	const char *dlid;
+	struct dm_info info;
+	uint64_t start, length;
+	struct dm_task *dmt = NULL;
+	char *type = NULL;
+	char *params = NULL;
+	int r = 0;
+	unsigned version = 0;
+
+	memset(cfg, 0, sizeof(*cfg));
+
+	if (!(dlid = build_dm_uuid(dm->mem, lv, lv_layer(lv))))
+		return_0;
+
+	if (!(dmt = _setup_task_run(DM_DEVICE_TABLE, &info, NULL, dlid, 0, 0, 0, 0, 0, 0)))
+		return_0;
+
+	if (!info.exists)
+		goto inactive; /* VDO device is not active, should not happen here... */
+
+	log_debug_activation("Checking VDO pool table line for LV %s.",
+			     display_lvname(lv));
+
+	if (dm_get_next_target(dmt, NULL, &start, &length, &type, &params)) {
+		log_error("More then one table line found for %s.",
+			  display_lvname(lv));
+		goto out;
+	}
+
+	if (!type || strcmp(type, TARGET_NAME_VDO)) {
+		log_error("Expected %s segment type but got %s instead.",
+			  TARGET_NAME_VDO, type ? type : "NULL");
+		goto out;
+	}
+
+	if (sscanf(params, "V%u %*s " FMTu64 " %*u " FMTu32,
+		   &version, &cfg->physical_size, &cfg->block_map_cache_size_mb) != 3) {
+		log_error("Failed to parse VDO parameters %s for LV %s.",
+			  params, display_lvname(lv));
+		goto out;
+	}
+
+	switch (version) {
+	case 2: break;
+	case 4: break;
+	default: log_warn("WARNING: Unknown VDO table line version %u.", version);
+	}
+
+	cfg->virtual_size = length;
+	cfg->physical_size *= 8; // From 4K unit to 512B
+	cfg->block_map_cache_size_mb /= 256; // From 4K unit to MiB
+	cfg->index_memory_size_mb = first_seg(lv)->vdo_params.index_memory_size_mb; // Preserved
+
+inactive:
+	r = 1;
+out:
+	dm_task_destroy(dmt);
+
+	return r;
+}
+
 
 /*************************/
 /*  NEW CODE STARTS HERE */
@@ -2198,6 +2321,7 @@ static int _add_dev_to_dtree(struct dev_manager *dm, struct dm_tree *dtree,
 {
 	char *dlid, *name;
 	struct dm_info info, info2;
+	const struct dm_active_device *dev;
 
 	if (!(name = dm_build_dm_name(dm->mem, lv->vg->name, lv->name, layer)))
 		return_0;
@@ -2205,9 +2329,21 @@ static int _add_dev_to_dtree(struct dev_manager *dm, struct dm_tree *dtree,
 	if (!(dlid = build_dm_uuid(dm->track_pending_delete ? dm->cmd->pending_delete_mem : dm->mem, lv, layer)))
 		return_0;
 
-	if (!_info(dm->cmd, name, dlid, 1, 0, 0, &info, NULL, NULL))
+	if (!dm->cmd->disable_dm_devs &&
+	    dm->cmd->cache_dm_devs) {
+		if (!dm_device_list_find_by_uuid(dm->cmd->cache_dm_devs, dlid, &dev)) {
+			log_debug("Cached as not present %s.", name);
+			return 1;
+		}
+		info = (struct dm_info) {
+			.exists = 1,
+			.major = dev->major,
+			.minor = dev->minor,
+		};
+		log_debug("Cached as present %s %s (%d:%d).",
+			  name, dlid, info.major, info.minor);
+	} else if (!_info(dm->cmd, name, dlid, 0, 0, 0, &info, NULL, NULL))
 		return_0;
-
 	/*
 	 * For top level volumes verify that existing device match
 	 * requested major/minor and that major/minor pair is available for use
@@ -2350,6 +2486,9 @@ static int _pool_callback(struct dm_tree_node *node,
 			return 0;
 		}
 	}
+
+	dm_device_list_destroy(&cmd->cache_dm_devs); /* Cache no longer valid */
+
 	log_debug("Running check command on %s", mpath);
 
 	if (data->skip_zero) {
@@ -2875,6 +3014,10 @@ int add_areas_line(struct dev_manager *dm, struct lv_segment *seg,
 
 	/* FIXME Avoid repeating identical stat in dm_tree_node_add_target_area */
 	for (s = start_area; s < areas; s++) {
+
+		/* FIXME: dev_name() does not return NULL!  It needs to check if dm_list_empty(&dev->aliases)
+		   but this knot of logic is too complex to pull apart without careful deconstruction. */
+
 		if ((seg_type(seg, s) == AREA_PV &&
 		     (!seg_pvseg(seg, s) || !seg_pv(seg, s) || !seg_dev(seg, s) ||
 		       !(name = dev_name(seg_dev(seg, s))) || !*name ||
@@ -2893,7 +3036,10 @@ int add_areas_line(struct dev_manager *dm, struct lv_segment *seg,
 				return_0;
 			num_error_areas++;
 		} else if (seg_type(seg, s) == AREA_PV) {
-			if (!dm_tree_node_add_target_area(node, dev_name(seg_dev(seg, s)), NULL,
+			struct device *dev = seg_dev(seg, s);
+			name = dm_list_empty(&dev->aliases) ? NULL : dev_name(dev);
+
+			if (!dm_tree_node_add_target_area(node, name, NULL,
 				    (seg_pv(seg, s)->pe_start + (extent_size * seg_pe(seg, s)))))
 				return_0;
 			num_existing_areas++;
@@ -3737,6 +3883,7 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv,
 	struct dm_tree_node *root;
 	char *dlid;
 	int r = 0;
+	unsigned tmp_state;
 
 	if (action < DM_ARRAY_SIZE(_action_names))
 		log_debug_activation("Creating %s%s tree for %s.",
@@ -3756,7 +3903,16 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv,
 	dm->suspend = (action == SUSPEND_WITH_LOCKFS) || (action == SUSPEND);
 	dm->track_external_lv_deps = 1;
 
-	if (!(dtree = _create_partial_dtree(dm, lv, laopts->origin_only)))
+	/* ATM do not use caching for anything else then striped target.
+	 * And also skip for CLEAN action */
+	tmp_state = dm->cmd->disable_dm_devs;
+	if (!seg_is_striped_target(first_seg(lv)) || (action == CLEAN))
+		dm->cmd->disable_dm_devs = 1;
+
+	dtree = _create_partial_dtree(dm, lv, laopts->origin_only);
+	dm->cmd->disable_dm_devs = tmp_state;
+
+	if (!dtree)
 		return_0;
 
 	if (!(root = dm_tree_find_node(dtree, 0, 0))) {
@@ -3819,6 +3975,8 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv,
 		 * non 'thin pool/volume' and  size increase */
 		else if (!lv_is_thin_volume(lv) &&
 			 !lv_is_thin_pool(lv) &&
+			 !lv_is_vdo(lv) &&
+			 !lv_is_vdo_pool(lv) &&
 			 dm_tree_node_size_changed(root))
 			dm->flush_required = 1;
 
@@ -3937,3 +4095,78 @@ out:
 
 	return r;
 }
+
+/*
+ * crypt offset is usually the LUKS header size but can be larger.
+ * The LUKS header is usually 2MB for LUKS1 and 16MB for LUKS2.
+ * The offset needs to be subtracted from the LV size to get the
+ * size used to resize the crypt device.
+ */
+int get_crypt_table_offset(dev_t crypt_devt, uint32_t *offset_bytes)
+{
+	struct dm_task *dmt = dm_task_create(DM_DEVICE_TABLE);
+	uint64_t start, length;
+	char *target_type = NULL;
+	void *next = NULL;
+	char *params = NULL;
+	char offset_str[32] = { 0 };
+	int copy_offset = 0;
+	int spaces = 0;
+	int i, i_off = 0;
+
+	if (!dmt)
+		return_0;
+
+	if (!dm_task_set_major_minor(dmt, (int)MAJOR(crypt_devt), (int)MINOR(crypt_devt), 0)) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	/* Non-blocking status read */
+	if (!dm_task_no_flush(dmt))
+		log_warn("WARNING: Can't set no_flush for dm status.");
+
+	if (!dm_task_run(dmt)) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	next = dm_get_next_target(dmt, next, &start, &length, &target_type, &params);
+
+	if (!target_type || !params || strcmp(target_type, "crypt")) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	/*
+	 * get offset from params string:
+	 * <cipher> <key> <iv_offset> <device> <offset> [<#opt_params> <opt_params>]
+	 * <offset> is reported in 512 byte sectors.
+	 */
+	for (i = 0; i < strlen(params); i++) {
+		if (params[i] == ' ') {
+			spaces++;
+			if (spaces == 4)
+				copy_offset = 1;
+			if (spaces == 5)
+				break;
+			continue;
+		}
+		if (!copy_offset)
+			continue;
+
+		offset_str[i_off++] = params[i];
+
+		if (i_off == sizeof(offset_str)) {
+			offset_str[0] = '\0';
+			break;
+		}
+	}
+	dm_task_destroy(dmt);
+
+	if (!offset_str[0])
+		return_0;
+
+	*offset_bytes = ((uint32_t)strtoul(offset_str, NULL, 0) * 512);
+	return 1;
+}

lib/activate/dev_manager.h

@@ -29,6 +29,8 @@ struct lv_seg_status;
 
 int read_only_lv(const struct logical_volume *lv, const struct lv_activate_opts *laopts, const char *layer);
 
+int get_crypt_table_offset(dev_t crypt_devt, uint32_t *offset_bytes);
+
 /*
  * Constructor and destructor.
  */
@@ -81,6 +83,9 @@ int dev_manager_thin_pool_status(struct dev_manager *dm,
 int dev_manager_vdo_pool_status(struct dev_manager *dm,
 				const struct logical_volume *lv, int flush,
 				struct lv_status_vdo **status, int *exists);
+int dev_manager_vdo_pool_size_config(struct dev_manager *dm,
+				     const struct logical_volume *lv,
+				     struct vdo_pool_size_config *cfg);
 int dev_manager_suspend(struct dev_manager *dm, const struct logical_volume *lv,
 			struct lv_activate_opts *laopts, int lockfs, int flush_required);
 int dev_manager_activate(struct dev_manager *dm, const struct logical_volume *lv,
@@ -103,5 +108,7 @@ int dev_manager_device_uses_vg(struct device *dev,
 int dev_manager_remove_dm_major_minor(uint32_t major, uint32_t minor);
 
 int dev_manager_check_prefix_dm_major_minor(uint32_t major, uint32_t minor, const char *prefix);
+int dev_manager_get_device_list(const char *prefix, struct dm_list **devs,
+				unsigned *devs_features);
 
 #endif

lib/activate/fs.c

@@ -318,9 +318,10 @@ struct fs_op_parms {
 
 static void _store_str(char **pos, char **ptr, const char *str)
 {
-	strcpy(*pos, str);
+	size_t len = strlen(str) + 1;
+	memcpy(*pos, str, len);
 	*ptr = *pos;
-	*pos += strlen(*ptr) + 1;
+	*pos += len;
 }
 
 static void _del_fs_op(struct fs_op_parms *fsp)

lib/cache/lvmcache.c

@@ -144,28 +144,6 @@ int lvmcache_found_duplicate_vgnames(void)
 	return _found_duplicate_vgnames;
 }
 
-static struct device_list *_get_devl_in_device_list(struct device *dev, struct dm_list *head)
-{
-	struct device_list *devl;
-
-	dm_list_iterate_items(devl, head) {
-		if (devl->dev == dev)
-			return devl;
-	}
-	return NULL;
-}
-
-int dev_in_device_list(struct device *dev, struct dm_list *head)
-{
-	struct device_list *devl;
-
-	dm_list_iterate_items(devl, head) {
-		if (devl->dev == dev)
-			return 1;
-	}
-	return 0;
-}
-
 bool lvmcache_has_duplicate_devs(void)
 {
 	if (dm_list_empty(&_unused_duplicates) && dm_list_empty(&_initial_duplicates))
@@ -192,11 +170,11 @@ void lvmcache_del_dev_from_duplicates(struct device *dev)
 {
 	struct device_list *devl;
 
-	if ((devl = _get_devl_in_device_list(dev, &_initial_duplicates))) {
+	if ((devl = device_list_find_dev(&_initial_duplicates, dev))) {
 		log_debug_cache("delete dev from initial duplicates %s", dev_name(dev));
 		dm_list_del(&devl->list);
 	}
-	if ((devl = _get_devl_in_device_list(dev, &_unused_duplicates))) {
+	if ((devl = device_list_find_dev(&_unused_duplicates, dev))) {
 		log_debug_cache("delete dev from unused duplicates %s", dev_name(dev));
 		dm_list_del(&devl->list);
 	}
@@ -354,9 +332,11 @@ static struct lvmcache_vginfo *_vginfo_lookup(const char *vgname, const char *vg
 	if (vgid_arg) {
 		if ((vginfo = dm_hash_lookup(_vgid_hash, vgid))) {
 			if (vgname && strcmp(vginfo->vgname, vgname)) {
-				/* should never happen */
-				log_error(INTERNAL_ERROR "vginfo_lookup vgid %s has two names %s %s",
-					  vgid, vginfo->vgname, vgname);
+				log_warn("WARNING: lookup found duplicate VGID %s for VGs %s and %s.", vgid, vginfo->vgname, vgname);
+				if ((vginfo = dm_hash_lookup(_vgname_hash, vgname))) {
+					if (!memcmp(vginfo->vgid, vgid, ID_LEN))
+						return vginfo;
+				}
 				return NULL;
 			}
 			return vginfo;
@@ -572,6 +552,16 @@ static const char *_get_pvsummary_device_id(const char *pvid_arg, const char **d
 	return NULL;
 }
 
+int lvmcache_pvsummary_count(const char *vgname)
+{
+	struct lvmcache_vginfo *vginfo;
+
+	if (!(vginfo = lvmcache_vginfo_from_vgname(vgname, NULL)))
+		return_0;
+
+	return dm_list_size(&vginfo->pvsummaries);
+}
+
 /*
  * Check if any PVs in vg->pvs have the same PVID as any
  * entries in _unused_duplicates.
@@ -593,7 +583,7 @@ int vg_has_duplicate_pvs(struct volume_group *vg)
 
 bool lvmcache_dev_is_unused_duplicate(struct device *dev)
 {
-	return dev_in_device_list(dev, &_unused_duplicates) ? true : false;
+	return device_list_find_dev(&_unused_duplicates, dev) ? true : false;
 }
 
 static void _warn_unused_duplicates(struct cmd_context *cmd)
@@ -625,6 +615,167 @@ static void _warn_unused_duplicates(struct cmd_context *cmd)
 	}
 }
 
+static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_info *info, const char *pvid,
+				     struct dm_list *altdevs, struct device **dev_mpath)
+{
+	struct device_list *devl;
+	struct device *dev_mp = NULL;
+	struct device *dev1 = NULL;
+	struct device *dev;
+	char wwid1_buf[DEV_WWID_SIZE] = { 0 };
+	char wwid_buf[DEV_WWID_SIZE] = { 0 };
+	const char *wwid1 = NULL;
+	const char *wwid = NULL;
+	int diff_wwid = 0;
+	int same_wwid = 0;
+	int dev_is_mp;
+
+	*dev_mpath = NULL;
+
+	if (!find_config_tree_bool(cmd, devices_multipath_component_detection_CFG, NULL))
+		return 0;
+
+	/* This function only makes sense with more than one dev. */
+	if ((info && dm_list_empty(altdevs)) || (!info && (dm_list_size(altdevs) == 1))) {
+		log_debug("Skip multipath component checks with single device for PVID %s", pvid);
+		return 0;
+	}
+
+	log_debug("Checking for multipath components for duplicate PVID %s", pvid);
+
+	if (info) {
+		dev = info->dev;
+		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
+
+		/*
+		 * dev_mpath_component_wwid allocates wwid from dm_pool,
+		 * device_id_system_read does not and needs free.
+		 */
+
+		if (dev_is_mp) {
+			if ((wwid1 = dev_mpath_component_wwid(cmd, dev))) {
+				strncpy(wwid1_buf, wwid1, DEV_WWID_SIZE-1);
+				dev_mp = dev;
+				dev1 = dev;
+			}
+		} else {
+			if ((wwid1 = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID))) {
+				strncpy(wwid1_buf, wwid1, DEV_WWID_SIZE-1);
+				free((char *)wwid1);
+				dev1 = dev;
+			}
+		}
+	}
+
+	dm_list_iterate_items(devl, altdevs) {
+		dev = devl->dev;
+		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
+
+		if (dev_is_mp) {
+			if ((wwid = dev_mpath_component_wwid(cmd, dev)))
+				strncpy(wwid_buf, wwid, DEV_WWID_SIZE-1);
+		} else {
+			if ((wwid = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID))) {
+				strncpy(wwid_buf, wwid, DEV_WWID_SIZE-1);
+				free((char *)wwid);
+			}
+		}
+
+		if (!wwid_buf[0] && wwid1_buf[0]) {
+			log_debug("Different wwids for duplicate PVs %s %s %s none",
+				  dev_name(dev1), wwid1_buf, dev_name(dev));
+			diff_wwid++;
+			continue;
+		}
+
+		if (!wwid_buf[0])
+			continue;
+
+		if (!wwid1_buf[0]) {
+			memcpy(wwid1_buf, wwid_buf, DEV_WWID_SIZE-1);
+			dev1 = dev;
+			continue;
+		}
+
+		/* Different wwids indicates these are not multipath components. */
+		if (strcmp(wwid1_buf, wwid_buf)) {
+			log_debug("Different wwids for duplicate PVs %s %s %s %s",
+				  dev_name(dev1), wwid1_buf, dev_name(dev), wwid_buf);
+			diff_wwid++;
+			continue;
+		}
+
+		/* Different mpath devs with the same wwid shouldn't happen. */
+		if (dev_is_mp && dev_mp) {
+			log_print("Found multiple multipath devices for PVID %s WWID %s: %s %s",
+				   pvid, wwid1_buf, dev_name(dev_mp), dev_name(dev));
+			continue;
+		}
+
+		log_debug("Same wwids for duplicate PVs %s %s", dev_name(dev1), dev_name(dev));
+		same_wwid++;
+
+		/* Save the mpath device so it can be used as the PV. */
+		if (dev_is_mp)
+			dev_mp = dev;
+	}
+
+	if (diff_wwid || !same_wwid)
+		return 0;
+
+	if (dev_mp)
+		log_debug("Found multipath device %s for PVID %s WWID %s.", dev_name(dev_mp), pvid, wwid1_buf);
+
+	*dev_mpath = dev_mp;
+	return 1;
+}
+
+static int _all_md_components(struct cmd_context *cmd, struct lvmcache_info *info, const char *pvid,
+			      struct dm_list *altdevs, struct device **dev_md_out)
+{
+	struct device_list *devl;
+	struct device *dev_md = NULL;
+	struct device *dev;
+	int real_dup = 0;
+ 
+	*dev_md_out = NULL;
+
+	/* There will often be no info struct because of the extra_md_checks function. */
+ 
+	if (info && (cmd->dev_types->md_major == MAJOR(info->dev->dev)))
+		dev_md = info->dev;
+ 
+	dm_list_iterate_items(devl, altdevs) {
+		dev = devl->dev;
+ 
+		if (cmd->dev_types->md_major == MAJOR(dev->dev)) {
+			if (dev_md) {
+				/* md devs themselves are dups */
+				log_debug("Found multiple md devices for PVID %s: %s %s",
+					  pvid, dev_name(dev_md), dev_name(dev));
+				real_dup = 1;
+				break;
+			} else
+				dev_md = dev;
+		} else {
+			if (!dev_is_md_component(cmd, dev, NULL, 1)) {
+				/* md dev copied to another device */
+				real_dup = 1;
+				break;
+			}
+		}
+	}
+ 
+	if (real_dup)
+		return 0;
+ 
+	if (dev_md)
+		log_debug("Found md device %s for PVID %s.", dev_name(dev_md), pvid);
+ 
+	*dev_md_out = dev_md;
+	return 1;
+}
+
 /*
  * If we've found devices with the same PVID, decide which one
  * to use.
@@ -680,6 +831,8 @@ static void _choose_duplicates(struct cmd_context *cmd,
 	struct device_list *devl, *devl_safe, *devl_add, *devl_del;
 	struct lvmcache_info *info;
 	struct device *dev1, *dev2;
+	struct device *dev_mpath, *dev_md;
+	struct device *dev_drop;
 	const char *device_id = NULL, *device_id_type = NULL;
 	const char *idname1 = NULL, *idname2 = NULL;
 	uint32_t dev1_major, dev1_minor, dev2_major, dev2_minor;
@@ -702,6 +855,8 @@ static void _choose_duplicates(struct cmd_context *cmd,
 next:
 	dm_list_init(&altdevs);
 	pvid = NULL;
+	dev_mpath = NULL;
+	dev_md = NULL;
 
 	dm_list_iterate_items_safe(devl, devl_safe, &_initial_duplicates) {
 		if (!pvid) {
@@ -720,30 +875,173 @@ next:
 		return;
 	}
 
+	info = lvmcache_info_from_pvid(pvid, NULL, 0);
+
 	/*
-	 * Get rid of any md components before comparing alternatives.
-	 * (Since an md component can never be used, it's not an
-	 * option to use like other kinds of alternatives.)
+	 * Usually and ideally, components of md and multipath devs should have
+	 * been excluded by filters, and not scanned for a PV.  In some unusual
+	 * cases the components can get through the filters, and a PV can be
+	 * found on them.  Detecting the same PVID on both the component and
+	 * the md/mpath device gives us a last chance to drop the component.
+	 * An md/mpath component device is completely ignored, as if it had
+	 * been filtered, and not kept in the list unused duplicates.
+	 *
+	 * One issue related to eliminating mpath/md duplicate PVs here is
+	 * that it occurs after label_scan, and hints are created based
+	 * on what label_scan finds, so hints are disabled due to duplicate
+	 * PVs that are later resolved here.
 	 */
 
-	info = lvmcache_info_from_pvid(pvid, NULL, 0);
-	if (info && dev_is_md_component(cmd, info->dev, NULL, 1)) {
-		/* does not go in del_cache_devs which become unused_duplicates */
-		log_debug_cache("PV %s drop MD component from scan selection %s", pvid, dev_name(info->dev));
-		lvmcache_del(info);
-		info = NULL;
-	}
+	/*
+	 * Get rid of multipath components based on matching wwids.
+	 */
+	if (_all_multipath_components(cmd, info, pvid, &altdevs, &dev_mpath)) {
+		if (info && dev_mpath && (info->dev != dev_mpath)) {
+			/*
+			 * info should be dropped from lvmcache and info->dev
+			 * should be treated as if it had been excluded by a filter.
+			 * dev_mpath should be added to lvmcache by the caller.
+			 */
+			dev_drop = info->dev;
 
-	dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
-		if (dev_is_md_component(cmd, devl->dev, NULL, 1)) {
-			log_debug_cache("PV %s drop MD component from scan duplicates %s", pvid, dev_name(devl->dev));
-			dm_list_del(&devl->list);
+			/* Have caller add dev_mpath to lvmcache. */
+			log_debug("Using multipath device %s for PVID %s.", dev_name(dev_mpath), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_mpath;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_mpath from altdevs. */
+			if ((devl = device_list_find_dev(&altdevs, dev_mpath)))
+				dm_list_del(&devl->list);
+
+			/* Remove info from lvmcache that came from the component dev. */
+			log_debug("Ignoring multipath component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
 		}
+
+		if (info && !dev_mpath) {
+			/*
+			 * Only mpath component devs were found and no actual
+			 * multipath dev, so drop the component from lvmcache.
+			 */
+			dev_drop = info->dev;
+
+			log_debug("Ignoring multipath component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
+			/*
+			 * The altdevs are all mpath components that should look
+			 * like they were filtered, they are not in lvmcache.
+			 */
+			dev_drop = devl->dev;
+
+			log_debug("Ignoring multipath component %s with PVID %s (dropping duplicate)", dev_name(dev_drop), pvid);
+			dm_list_del(&devl->list);
+
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+		goto next;
 	}
 
-	if (dm_list_empty(&altdevs))
-		goto next;
+	/*
+	 * Get rid of any md components.
+	 */
+	if (_all_md_components(cmd, info, pvid, &altdevs, &dev_md)) {
+		if (info && dev_md && (info->dev != dev_md)) {
+			/*
+			 * info should be dropped from lvmcache and info->dev
+			 * should be treated as if it had been excluded by a filter.
+			 * dev_md should be added to lvmcache by the caller.
+			 * Often this info struct has been removed by
+			 * lvmcache_extra_md_component_checks.
+			 */
+			dev_drop = info->dev;
 
+			/* Have caller add dev_md to lvmcache. */
+			log_debug("Using md device %s for PVID %s.", dev_name(dev_md), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_md;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_md from altdevs. */
+			if ((devl = device_list_find_dev(&altdevs, dev_md)))
+				dm_list_del(&devl->list);
+
+			/* Remove info from lvmcache that came from the component dev. */
+			log_debug("Ignoring md component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		if (!info && dev_md) {
+			/*
+			 * The info struct was from a component and was dropped
+			 * and the actual md dev was found on initial_duplicates
+			 * and the caller should add it to lvmcache.
+			 */
+
+			/* Have caller add dev_md to lvmcache. */
+			log_debug("Using md device %s for PVID %s.", dev_name(dev_md), pvid);
+			if ((devl_add = zalloc(sizeof(*devl_add)))) {
+				devl_add->dev = dev_md;
+				dm_list_add(add_cache_devs, &devl_add->list);
+			}
+
+			/* Remove dev_md from altdevs. */
+			if ((devl = device_list_find_dev(&altdevs, dev_md)))
+				dm_list_del(&devl->list);
+		}
+
+		if (info && !dev_md) {
+			/*
+			 * Only md component devs were found and no actual
+			 * md dev, so drop the component from lvmcache.
+			 */
+			dev_drop = info->dev;
+
+			log_debug("Ignoring md component %s with PVID %s (dropping info)", dev_name(dev_drop), pvid);
+			lvmcache_del(info);
+			info = NULL;
+
+			/* Make the component dev look like it was filtered. */
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+
+		dm_list_iterate_items_safe(devl, devl_safe, &altdevs) {
+			/*
+			 * The altdevs are all md components that should look
+			 * like they were filtered, they are not in lvmcache.
+			 */
+			dev_drop = devl->dev;
+
+			log_debug("Ignoring md component %s with PVID %s (dropping duplicate)", dev_name(dev_drop), pvid);
+			dm_list_del(&devl->list);
+
+			cmd->filter->wipe(cmd, cmd->filter, dev_drop, NULL);
+			dev_drop->flags &= ~DEV_SCAN_FOUND_LABEL;
+		}
+		goto next;
+	}
 
 	/*
 	 * Find the device for the pvid that's currently in lvmcache.
@@ -787,8 +1085,8 @@ next:
 		if (dev1 == dev2)
 			continue;
 
-		prev_unchosen1 = dev_in_device_list(dev1, &_unused_duplicates);
-		prev_unchosen2 = dev_in_device_list(dev2, &_unused_duplicates);
+		prev_unchosen1 = device_list_find_dev(&_unused_duplicates, dev1) ? 1 :0;
+		prev_unchosen2 = device_list_find_dev(&_unused_duplicates, dev2) ? 1 :0;
 
 		if (!prev_unchosen1 && !prev_unchosen2) {
 			/*
@@ -798,8 +1096,8 @@ next:
 			 * want the same duplicate preference to be preserved
 			 * in each instance of lvmcache for a single command.
 			 */
-			prev_unchosen1 = dev_in_device_list(dev1, &_prev_unused_duplicate_devs);
-			prev_unchosen2 = dev_in_device_list(dev2, &_prev_unused_duplicate_devs);
+			prev_unchosen1 = device_list_find_dev(&_prev_unused_duplicate_devs, dev1) ? 1 :0;
+			prev_unchosen2 = device_list_find_dev(&_prev_unused_duplicate_devs, dev2) ? 1 : 0;
 		}
 
 		dev1_major = MAJOR(dev1->dev);
@@ -976,7 +1274,7 @@ next:
 	if (!info) {
 		log_debug_cache("PV %s with duplicates will use %s.", pvid, dev_name(dev1));
 
-		if (!(devl_add = _get_devl_in_device_list(dev1, &altdevs))) {
+		if (!(devl_add = device_list_find_dev(&altdevs, dev1))) {
 			/* shouldn't happen */
 			log_error(INTERNAL_ERROR "PV %s with duplicates no alternate list entry for %s", pvid, dev_name(dev1));
 			dm_list_splice(&new_unused, &altdevs);
@@ -995,7 +1293,7 @@ next:
 		 * for the current lvmcache device to drop.
 		 */
 
-		if (!(devl_add = _get_devl_in_device_list(dev1, &altdevs))) {
+		if (!(devl_add = device_list_find_dev(&altdevs, dev1))) {
 			/* shouldn't happen */
 			log_error(INTERNAL_ERROR "PV %s with duplicates no alternate list entry for %s", pvid, dev_name(dev1));
 			dm_list_splice(&new_unused, &altdevs);
@@ -1149,6 +1447,18 @@ int lvmcache_label_reopen_vg_rw(struct cmd_context *cmd, const char *vgname, con
  * times it can be a clue that label_scan mistakenly read the pv from an md
  * component device instead of from the md device itself.  So for unmatching
  * sizes, we do a full md component check on the device.
+ *
+ * It might be nice to do this checking in the filter (when passes_filter is
+ * called after the initial read), but that doesn't work because passes_filter
+ * is called before _text_read so metadata/pvsummary info is not yet available
+ * which this function uses.
+ *
+ * The unique value of this function is that it can eliminate md components
+ * without there being duplicate PVs.  But, there will often be duplicate PVs,
+ * handled by _all_md_components(), where other devs with the same pvid will be
+ * in _initial_duplicates.  One could be the md device itself which will be
+ * added to lvmcache by choose_duplicates, and other duplicates that are
+ * components will be dropped.
  */
 
 void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
@@ -1210,7 +1520,8 @@ void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
 			 */
 			if (pvsize && devsize && (pvsize != devsize))
 				do_check_size = 1;
-			if (device_hint && !strncmp(device_hint, "/dev/md", 7))
+			if (device_hint && !strncmp(device_hint, "/dev/md", 7) &&
+			    (MAJOR(info->dev->dev) != cmd->dev_types->md_major))
 				do_check_name = 1;
 
 			if (!do_check_size && !do_check_name)
@@ -1240,11 +1551,11 @@ void lvmcache_extra_md_component_checks(struct cmd_context *cmd)
 				  device_hint ?: "none", dev_name(dev));
 
 			if (dev_is_md_component(cmd, dev, NULL, 1)) {
-				log_debug("dropping PV from md component %s", dev_name(dev));
+				log_debug("Ignoring PV from md component %s with PVID %s (metadata %s %llu)",
+					  dev_name(dev), dev->pvid, device_hint ?: "none", (unsigned long long)pvsize);
 				dev->flags &= ~DEV_SCAN_FOUND_LABEL;
 				/* lvmcache_del will also delete vginfo if info was last one */
 				lvmcache_del(info);
-				lvmcache_del_dev_from_duplicates(dev);
 				cmd->filter->wipe(cmd, cmd->filter, dev, NULL);
 			}
 		}
@@ -1298,7 +1609,24 @@ int lvmcache_label_scan(struct cmd_context *cmd)
 	 * with infos/vginfos based on reading headers from
 	 * each device, and a vg summary from each mda.
 	 */
-	label_scan(cmd);
+	if (!label_scan(cmd))
+		return_0;
+
+	/*
+	 * device_ids_validate() found devices using a sys_serial device id
+	 * which had a PVID on disk that did not match the PVID in the devices
+	 * file.  Serial numbers may not always be unique, so any device with
+	 * the same serial number is found and searched for the correct PVID.
+	 * If the PVID is found on a device that has not been scanned, then
+	 * it needs to be scanned so it can be used.
+	 */
+	if (!dm_list_empty(&cmd->device_ids_check_serial)) {
+		struct dm_list scan_devs;
+		dm_list_init(&scan_devs);
+		device_ids_check_serial(cmd, &scan_devs, NULL, 0);
+		if (!dm_list_empty(&scan_devs))
+			label_scan_devs(cmd, cmd->filter, &scan_devs);
+	}
 
 	/*
 	 * When devnames are used as device ids (which is dispreferred),
@@ -1572,7 +1900,17 @@ static int _lvmcache_update_vgname(struct cmd_context *cmd,
 
 	_drop_vginfo(info, info->vginfo);
 
-	if (!(vginfo = lvmcache_vginfo_from_vgid(vgid))) {
+	vginfo = lvmcache_vginfo_from_vgid(vgid);
+	if (vginfo && strcmp(vginfo->vgname, vgname)) {
+		log_warn("WARNING: fix duplicate VGID %s for VGs %s and %s (see vgchange -u).", vgid_dashed, vgname, vginfo->vgname);
+		vginfo = lvmcache_vginfo_from_vgname(vgname, NULL);
+		if (vginfo && memcmp(vginfo->vgid, vgid, ID_LEN)) {
+			log_error("Ignoring %s with conflicting VG info %s %s.", dev_name(info->dev), vgid_dashed, vgname);
+			return_0;
+		}
+	}
+
+	if (!vginfo) {
 		/*
 	 	 * Create a vginfo struct for this VG and put the vginfo
 	 	 * into the hash table.
@@ -1927,57 +2265,13 @@ int lvmcache_update_vgname_and_id(struct cmd_context *cmd, struct lvmcache_info
 	return 1;
 }
 
-/*
- * FIXME: quit trying to mirror changes that a command is making into lvmcache.
- *
- * First, it's complicated and hard to ensure it's done correctly in every case
- * (it would be much easier and safer to just toss out what's in lvmcache and
- * reread the info to recreate it from scratch instead of trying to make sure
- * every possible discrete state change is correct.)
- *
- * Second, it's unnecessary if commands just use the vg they are modifying
- * rather than also trying to get info from lvmcache.  The lvmcache state
- * should be populated by label_scan, used to perform vg_read's, and then
- * ignored (or dropped so it can't be used).
- *
- * lvmcache info is already used very little after a command begins its
- * operation.  The code that's supposed to keep the lvmcache in sync with
- * changes being made to disk could be half wrong and we wouldn't know it.
- * That creates a landmine for someone who might try to use a bit of it that
- * isn't being updated correctly.
- */
-
-int lvmcache_update_vg_from_write(struct volume_group *vg)
-{
-	char vgid[ID_LEN + 1] __attribute__((aligned(8))) = { 0 };
-	struct pv_list *pvl;
-	struct lvmcache_info *info;
-	struct lvmcache_vgsummary vgsummary = {
-		.vgname = vg->name,
-		.vgstatus = vg->status,
-		.system_id = vg->system_id,
-		.lock_type = vg->lock_type
-	};
-
-	memcpy(vgid, &vg->id, ID_LEN);
-	memcpy(vgsummary.vgid, vgid, ID_LEN);
-
-	dm_list_iterate_items(pvl, &vg->pvs) {
-		if ((info = lvmcache_info_from_pv_id(&pvl->pv->id, pvl->pv->dev, 0)) &&
-		    !lvmcache_update_vgname_and_id(vg->cmd, info, &vgsummary))
-			return_0;
-	}
-
-	return 1;
-}
-
 /*
  * The lvmcache representation of a VG after label_scan can be incorrect
  * because the label_scan does not use the full VG metadata to construct
  * vginfo/info.  PVs that don't hold VG metadata weren't attached to the vginfo
  * during label scan, and PVs with outdated metadata (claiming to be in the VG,
  * but not listed in the latest metadata) were attached to the vginfo, but
- * shouldn't be.  After vg_read() gets the full metdata in the form of a 'vg',
+ * shouldn't be.  After vg_read() gets the full metadata in the form of a 'vg',
  * this function is called to fix up the lvmcache representation of the VG
  * using the 'vg'.
  */
@@ -2214,7 +2508,7 @@ struct lvmcache_info *lvmcache_add(struct cmd_context *cmd, struct labeller *lab
 			memcpy(dev->pvid, pvid, ID_LEN);
 
 			/* shouldn't happen */
-			if (dev_in_device_list(dev, &_initial_duplicates))
+			if (device_list_find_dev(&_initial_duplicates, dev))
 				log_debug_cache("Initial duplicate already in list %s", dev_name(dev));
 			else {
 				/*

lib/cache/lvmcache.h

@@ -84,7 +84,6 @@ void lvmcache_del_dev(struct device *dev);
 int lvmcache_update_vgname_and_id(struct cmd_context *cmd, struct lvmcache_info *info,
 				  struct lvmcache_vgsummary *vgsummary);
 int lvmcache_update_vg_from_read(struct volume_group *vg, unsigned precommitted);
-int lvmcache_update_vg_from_write(struct volume_group *vg);
 
 void lvmcache_lock_vgname(const char *vgname, int read_only);
 void lvmcache_unlock_vgname(const char *vgname);
@@ -187,8 +186,6 @@ int lvmcache_vginfo_has_pvid(struct lvmcache_vginfo *vginfo, const char *pvid_ar
 uint64_t lvmcache_max_metadata_size(void);
 void lvmcache_save_metadata_size(uint64_t val);
 
-int dev_in_device_list(struct device *dev, struct dm_list *head);
-
 bool lvmcache_has_bad_metadata(struct device *dev);
 
 bool lvmcache_has_old_metadata(struct cmd_context *cmd, const char *vgname, const char *vgid, struct device *dev);
@@ -229,4 +226,6 @@ void lvmcache_extra_md_component_checks(struct cmd_context *cmd);
 
 unsigned int lvmcache_vg_info_count(void);
 
+int lvmcache_pvsummary_count(const char *vgname);
+
 #endif

lib/commands/cmd_enum.h

@@ -0,0 +1,19 @@
+#ifndef _CMD_ENUM_H
+#define _CMD_ENUM_H
+
+/*
+ * include/cmds.h is generated by the Makefile.  For each command definition
+ * in command-lines.in, cmds.h contains:
+ * cmd(foo_CMD, foo)
+ *
+ * This header adds each of the foo_CMD's into an enum, so there's
+ * a unique integer identifier for each command definition.
+ */
+
+enum {
+#define cmd(a, b) a ,
+#include "../../include/cmds.h"
+#undef cmd
+};
+
+#endif

lib/commands/toolcontext.c

@@ -159,9 +159,11 @@ static const char *_system_id_from_source(struct cmd_context *cmd, const char *s
 
 #ifdef APP_MACHINEID_SUPPORT
 	if (!strcasecmp(source, "appmachineid")) {
-		sd_id128_t id;
+		sd_id128_t id = { 0 };
 
-		sd_id128_get_machine_app_specific(LVM_APPLICATION_ID, &id);
+		if (sd_id128_get_machine_app_specific(LVM_APPLICATION_ID, &id) != 0)
+			log_warn("WARNING: sd_id128_get_machine_app_specific() failed %s (%d).",
+				 strerror(errno), errno);
 
 		if (dm_snprintf(buf, PATH_MAX, SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(id)) < 0)
 			stack;
@@ -641,6 +643,7 @@ static int _process_config(struct cmd_context *cmd)
 	if (!dm_set_uuid_prefix(UUID_PREFIX))
 		return_0;
 #endif
+	cmd->device_id_sysfs_dir = find_config_tree_str(cmd, devices_device_id_sysfs_dir_CFG, NULL);
 
 	dev_ext_info_src = find_config_tree_str(cmd, devices_external_device_info_source_CFG, NULL);
 
@@ -1126,7 +1129,7 @@ static int _init_dev_cache(struct cmd_context *cmd)
 	return 1;
 }
 
-#define MAX_FILTERS 11
+#define MAX_FILTERS 10
 
 static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 {
@@ -1141,26 +1144,6 @@ static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 	 * Update MAX_FILTERS definition above when adding new filters.
 	 */
 
-	/*
-	 * sysfs filter. Only available on 2.6 kernels.  Non-critical.
-	 * Listed first because it's very efficient at eliminating
-	 * unavailable devices.
-	 *
-	 * TODO: I suspect that using the lvm_type and device_id
-	 * filters before this one may be more efficient.
-	 */
-	if (find_config_tree_bool(cmd, devices_sysfs_scan_CFG, NULL)) {
-		if ((filters[nr_filt] = sysfs_filter_create()))
-			nr_filt++;
-	}
-
-	/* internal filter used by command processing. */
-	if (!(filters[nr_filt] = internal_filter_create())) {
-		log_error("Failed to create internal device filter");
-		goto bad;
-	}
-	nr_filt++;
-
 	/* global regex filter. Optional. */
 	if ((cn = find_config_tree_node(cmd, devices_global_filter_CFG, NULL))) {
 		if (!(filters[nr_filt] = regex_filter_create(cn->v, 0, 1))) {
@@ -1193,6 +1176,17 @@ static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 	}
 	nr_filt++;
 
+	/*
+	 * sysfs filter. Only available on 2.6 kernels.  Non-critical.
+	 * Eliminates unavailable devices.
+	 * TODO: this may be unnecessary now with device ids
+	 * (currently not used for devs match to device id using syfs)
+	 */
+	if (find_config_tree_bool(cmd, devices_sysfs_scan_CFG, NULL)) {
+		if ((filters[nr_filt] = sysfs_filter_create()))
+			nr_filt++;
+	}
+
 	/* usable device filter. Required. */
 	if (!(filters[nr_filt] = usable_filter_create(cmd, cmd->dev_types, FILTER_MODE_NO_LVMETAD))) {
 		log_error("Failed to create usabled device filter");
@@ -1587,7 +1581,7 @@ struct cmd_context *create_config_context(void)
 	if (!(cmd = zalloc(sizeof(*cmd))))
 		goto_out;
 
-	strcpy(cmd->system_dir, DEFAULT_SYS_DIR);
+	strncpy(cmd->system_dir, DEFAULT_SYS_DIR, sizeof(cmd->system_dir) - 1);
 
 	if (!_get_env_vars(cmd))
 		goto_out;
@@ -1603,7 +1597,6 @@ struct cmd_context *create_config_context(void)
 
 	dm_list_init(&cmd->config_files);
 	dm_list_init(&cmd->tags);
-	dm_list_init(&cmd->hints);
 
 	if (!_init_lvm_conf(cmd))
 		goto_out;
@@ -1668,7 +1661,6 @@ struct cmd_context *create_toolcontext(unsigned is_clvmd,
 	dm_list_init(&cmd->formats);
 	dm_list_init(&cmd->segtypes);
 	dm_list_init(&cmd->tags);
-	dm_list_init(&cmd->hints);
 	dm_list_init(&cmd->config_files);
 	label_init();
 
@@ -1721,10 +1713,8 @@ struct cmd_context *create_toolcontext(unsigned is_clvmd,
 	/*
 	 * Environment variable LVM_SYSTEM_DIR overrides this below.
 	 */
-        if (system_dir)
-		strncpy(cmd->system_dir, system_dir, sizeof(cmd->system_dir) - 1);
-	else
-		strcpy(cmd->system_dir, DEFAULT_SYS_DIR);
+	strncpy(cmd->system_dir, (system_dir) ? system_dir : DEFAULT_SYS_DIR,
+		sizeof(cmd->system_dir) - 1);
 
 	if (!_get_env_vars(cmd))
 		goto_out;
@@ -1914,7 +1904,6 @@ int refresh_toolcontext(struct cmd_context *cmd)
 	_destroy_segtypes(&cmd->segtypes);
 	_destroy_formats(cmd, &cmd->formats);
 
-	devices_file_exit(cmd);
 	if (!dev_cache_exit())
 		stack;
 	_destroy_dev_types(cmd);
@@ -2043,25 +2032,17 @@ void destroy_toolcontext(struct cmd_context *cmd)
 	_destroy_segtypes(&cmd->segtypes);
 	_destroy_formats(cmd, &cmd->formats);
 	_destroy_filters(cmd);
-	if (cmd->mem)
-		dm_pool_destroy(cmd->mem);
-	devices_file_exit(cmd);
 	dev_cache_exit();
 	_destroy_dev_types(cmd);
 	_destroy_tags(cmd);
 
 	if ((cft_cmdline = remove_config_tree_by_source(cmd, CONFIG_STRING)))
 		config_destroy(cft_cmdline);
-	_destroy_config(cmd);
 
 	if (cmd->cft_def_hash)
 		dm_hash_destroy(cmd->cft_def_hash);
 
-	if (cmd->libmem)
-		dm_pool_destroy(cmd->libmem);
-
-	if (cmd->pending_delete_mem)
-		dm_pool_destroy(cmd->pending_delete_mem);
+	dm_device_list_destroy(&cmd->cache_dm_devs);
 #ifndef VALGRIND_POOL
 	if (cmd->linebuffer) {
 		/* Reset stream buffering to defaults */
@@ -2086,7 +2067,7 @@ void destroy_toolcontext(struct cmd_context *cmd)
 		free(cmd->linebuffer);
 	}
 #endif
-	free(cmd);
+	destroy_config_context(cmd);
 
 	lvmpolld_disconnect();
 

lib/commands/toolcontext.h

@@ -18,6 +18,7 @@
 
 #include "lib/device/dev-cache.h"
 #include "lib/device/dev-type.h"
+#include "lib/commands/cmd_enum.h"
 
 #include <limits.h>
 
@@ -94,6 +95,7 @@ struct cmd_context {
 	const char *name; /* needed before cmd->command is set */
 	struct command_name *cname;
 	struct command *command;
+	int command_enum; /* duplicate from command->command_enum for lib code */
 	char **argv;
 	struct arg_values *opt_arg_values;
 	struct dm_list arg_value_groups;
@@ -144,6 +146,7 @@ struct cmd_context {
 	unsigned degraded_activation:1;
 	unsigned auto_set_activation_skip:1;
 	unsigned si_unit_consistency:1;
+	unsigned report_strict_type_mode:1;
 	unsigned report_binary_values_as_numeric:1;
 	unsigned report_mark_hidden_devices:1;
 	unsigned metadata_read_only:1;
@@ -174,7 +177,7 @@ struct cmd_context {
 	unsigned activate_component:1;		/* command activates component LV */
 	unsigned process_component_lvs:1;	/* command processes also component LVs */
 	unsigned mirror_warn_printed:1;		/* command already printed warning about non-monitored mirrors */
-	unsigned pvscan_cache_single:1;
+	unsigned expect_missing_vg_device:1;	/* when reading a vg it's expected that a dev for a pv isn't found */
 	unsigned can_use_one_scan:1;
 	unsigned is_clvmd:1;
 	unsigned md_component_detection:1;
@@ -191,6 +194,7 @@ struct cmd_context {
 	unsigned create_edit_devices_file:1;	/* command expects to create and/or edit devices file */
 	unsigned edit_devices_file:1;		/* command expects to edit devices file */
 	unsigned filter_deviceid_skip:1;	/* don't use filter-deviceid */
+	unsigned filter_regex_skip:1;		/* don't use filter-regex */
 	unsigned filter_regex_with_devices_file:1; /* use filter-regex even when devices file is enabled */
 	unsigned filter_nodata_only:1;          /* only use filters that do not require data from the dev */
 	unsigned run_by_dmeventd:1;		/* command is being run by dmeventd */
@@ -201,18 +205,24 @@ struct cmd_context {
 	unsigned ignore_device_name_mismatch:1; /* skip updating devices file names */
 	unsigned backup_disabled:1;		/* skip repeated debug message */
 	unsigned event_activation:1;		/* whether event_activation is set */
+	unsigned udevoutput:1;
+	unsigned online_vg_file_removed:1;
+	unsigned disable_dm_devs:1;		/* temporarily disable use of dm devs cache */
+	unsigned filter_regex_set_preferred_name_disable:1; /* prevent dev_set_preferred_name */
 
 	/*
 	 * Devices and filtering.
 	 */
 	struct dev_filter *filter;
-	struct dm_list hints;
 	struct dm_list use_devices;		/* struct dev_use for each entry in devices file */
 	const char *md_component_checks;
 	const char *search_for_devnames;	/* config file setting */
+	struct dm_list device_ids_check_serial;
 	const char *devicesfile;                /* from --devicesfile option */
 	struct dm_list deviceslist;             /* from --devices option, struct dm_str_list */
 
+	struct dm_list *cache_dm_devs;		/* cache with UUIDs from DM_DEVICE_LIST (when available) */
+
 	/*
 	 * Configuration.
 	 */
@@ -240,6 +250,7 @@ struct cmd_context {
 	 * Paths.
 	 */
 	const char *lib_dir;			/* cache value global/library_dir */
+	const char *device_id_sysfs_dir;
 	char system_dir[PATH_MAX];
 	char dev_dir[PATH_MAX];
 	char proc_dir[PATH_MAX];

lib/config/config.c

@@ -522,7 +522,9 @@ int config_file_read_fd(struct dm_config_tree *cft, struct device *dev, dev_io_r
 	if (!(dev->flags & DEV_REGULAR) || size2)
 		use_plain_read = 0;
 
-	if (!(buf = zalloc(size + size2))) {
+	/* Ensure there is extra '\0' after end of buffer since we pass
+	 * buffer to funtions like strtoll() */
+	if (!(buf = zalloc(size + size2 + 1))) {
 		log_error("Failed to allocate circular buffer.");
 		return 0;
 	}
@@ -1167,7 +1169,7 @@ int config_def_check(struct cft_check_handle *handle)
 	*vp = 0;
 	*rp = 0;
 	if (!handle->cmd->cft_def_hash) {
-		if (!(handle->cmd->cft_def_hash = dm_hash_create(60))) {
+		if (!(handle->cmd->cft_def_hash = dm_hash_create(500))) {
 			log_error("Failed to create configuration definition hash.");
 			r = 0; goto out;
 		}
@@ -1828,8 +1830,9 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 	char summary[MAX_COMMENT_LINE+1];
 	char version[9];
 	int pos = 0;
+	int space_prefix_len = 0;
+	const char *p;
 	size_t len;
-	char *space_prefix;
 
 	if ((out->tree_spec->type == CFG_DEF_TREE_DIFF) &&
 	    (!(out->tree_spec->check_status[cn->id] & CFG_DIFF)))
@@ -1863,16 +1866,36 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 
 	/* Usual tree view with nodes and their values. */
 
+	if (out->tree_spec->valuesonly && !(cfg_def->type & CFG_TYPE_SECTION)) {
+		if ((space_prefix_len = strspn(line, "\t "))) {
+			len = strlen(line);
+			p = line + space_prefix_len;
+
+			/* copy space_prefix, skip key and '=', copy value */
+			if (!dm_pool_begin_object(out->mem, len))
+				return_0;
+
+			if (!dm_pool_grow_object(out->mem, line, space_prefix_len) ||
+			    !dm_pool_grow_object(out->mem, p + strcspn(p, "=") + 1, len + 1)) {
+				dm_pool_abandon_object(out->mem);
+				return_0;
+			}
+
+			line = dm_pool_end_object(out->mem);
+		} else
+			line = strchr(line, '=') + 1;
+	}
+
 	if ((out->tree_spec->type != CFG_DEF_TREE_CURRENT) &&
 	    (out->tree_spec->type != CFG_DEF_TREE_DIFF) &&
 	    (out->tree_spec->type != CFG_DEF_TREE_FULL) &&
+	    !out->tree_spec->valuesonly &&
 	    (cfg_def->flags & (CFG_DEFAULT_UNDEFINED | CFG_DEFAULT_COMMENTED))) {
 		/* print with # at the front to comment out the line */
 		if (_should_print_cfg_with_undef_def_val(out, cfg_def, cn)) {
-			space_prefix = ((len = strspn(line, "\t "))) ? dm_pool_strndup(out->mem, line, len) : NULL;
-			fprintf(out->fp, "%s%s%s\n", space_prefix ? : "", "# ", line + len);
-			if (space_prefix)
-				dm_pool_free(out->mem, space_prefix);
+			space_prefix_len = strspn(line, "\t ");
+			fprintf(out->fp, "%.*s%s%s\n", space_prefix_len, line, "# ",
+				line + space_prefix_len);
 		}
 		return 1;
 	}
@@ -1881,6 +1904,9 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 	if (_should_print_cfg_with_undef_def_val(out, cfg_def, cn))
 		fprintf(out->fp, "%s\n", line);
 
+	if (out->tree_spec->valuesonly && !(cfg_def->type & CFG_TYPE_SECTION) && space_prefix_len)
+		dm_pool_free(out->mem, (char *) line);
+
 	return 1;
 }
 

lib/config/config.h

@@ -176,6 +176,7 @@ struct config_def_tree_spec {
 	unsigned unconfigured:1;		/* use unconfigured path strings */
 	unsigned withgeneralpreamble:1;		/* include preamble for a general config file */
 	unsigned withlocalpreamble:1;		/* include preamble for a local config file */
+	unsigned valuesonly:1;                  /* print only values without keys */
 	uint8_t *check_status;			/* status of last tree check (currently needed for CFG_DEF_TREE_MISSING only) */
 };
 

lib/config/config_settings.h

@@ -118,6 +118,7 @@
  * the previous default value was set (uncommented) in lvm.conf.
  */
 #include "lib/config/defaults.h"
+#include "device_mapper/vdo/vdo_limits.h"
 
 cfg_section(root_CFG_SECTION, "(root)", root_CFG_SECTION, 0, vsn(0, 0, 0), 0, NULL, NULL)
 
@@ -223,6 +224,9 @@ cfg(devices_dir_CFG, "dir", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_ADV
 	"Directory in which to create volume group device nodes.\n"
 	"Commands also accept this as a prefix on volume group names.\n")
 
+cfg(devices_device_id_sysfs_dir_CFG, "device_id_sysfs_dir", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_UNSUPPORTED, CFG_TYPE_STRING, DEFAULT_DEVICE_ID_SYSFS_DIR, vsn(2, 3, 17), NULL, 0, NULL,
+	"Location of sysfs for finding device ids (for testing.)\n")
+
 cfg_array(devices_scan_CFG, "scan", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_ADVANCED, CFG_TYPE_STRING, "#S/dev", vsn(1, 0, 0), NULL, 0, NULL,
 	"Directories containing device nodes to use with LVM.\n")
 
@@ -276,7 +280,7 @@ cfg_array(devices_preferred_names_CFG, "preferred_names", devices_CFG_SECTION, C
 	"preferred_names = [ \"^/dev/mpath/\", \"^/dev/mapper/mpath\", \"^/dev/[hs]d\" ]\n"
 	"#\n")
 
-cfg(devices_use_devicesfile_CFG, "use_devicesfile", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_USE_DEVICES_FILE, vsn(2, 3, 12), NULL, 0, NULL,
+cfg(devices_use_devicesfile_CFG, "use_devicesfile", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_USE_DEVICES_FILE, vsn(2, 3, 12), "@DEFAULT_USE_DEVICES_FILE@", 0, NULL,
 	"Enable or disable the use of a devices file.\n"
 	"When enabled, lvm will only use devices that\n"
 	"are lised in the devices file. A devices file will\n"
@@ -708,12 +712,11 @@ cfg(allocation_vdo_use_deduplication_CFG, "vdo_use_deduplication", allocation_CF
 cfg(allocation_vdo_use_metadata_hints_CFG, "vdo_use_metadata_hints", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_USE_METADATA_HINTS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Enables or disables whether VDO volume should tag its latency-critical\n"
 	"writes with the REQ_SYNC flag. Some device mapper targets such as dm-raid5\n"
-	"process writes with this flag at a higher priority.\n"
-	"Default is enabled.\n")
+	"process writes with this flag at a higher priority.\n")
 
 cfg(allocation_vdo_minimum_io_size_CFG, "vdo_minimum_io_size", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_MINIMUM_IO_SIZE, VDO_1ST_VSN, NULL, 0, NULL,
 	"The minimum IO size for VDO volume to accept, in bytes.\n"
-	"Valid values are 512 or 4096. The recommended and default value is 4096.\n")
+	"Valid values are 512 or 4096. The recommended value is 4096.\n")
 
 cfg(allocation_vdo_block_map_cache_size_mb_CFG, "vdo_block_map_cache_size_mb", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BLOCK_MAP_CACHE_SIZE_MB, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the amount of memory in MiB allocated for caching block map\n"
@@ -726,7 +729,8 @@ cfg(allocation_vdo_block_map_era_length_CFG, "vdo_block_map_period", allocation_
 	"The speed with which the block map cache writes out modified block map pages.\n"
 	"A smaller era length is likely to reduce the amount time spent rebuilding,\n"
 	"at the cost of increased block map writes during normal operation.\n"
-	"The maximum and recommended value is 16380; the minimum value is 1.\n")
+	"The maximum and recommended value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM)
+	"; the minimum value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM) ".\n")
 
 cfg(allocation_vdo_check_point_frequency_CFG, "vdo_check_point_frequency", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CHECK_POINT_FREQUENCY, VDO_1ST_VSN, NULL, 0, NULL,
 	"The default check point frequency for VDO volume.\n")
@@ -748,27 +752,34 @@ cfg(allocation_vdo_slab_size_mb_CFG, "vdo_slab_size_mb", allocation_CFG_SECTION,
 cfg(allocation_vdo_ack_threads_CFG, "vdo_ack_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_ACK_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for acknowledging\n"
 	"completion of requested VDO I/O operations.\n"
-	"The value must be at in range [0..100].\n")
+	"The value must be at in range [" DM_TO_STRING(DM_VDO_ACK_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_ACK_THREADS_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_bio_threads_CFG, "vdo_bio_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BIO_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for submitting I/O\n"
 	"operations to the storage device of VDO volume.\n"
-	"The value must be in range [1..100]\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_BIO_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_BIO_THREADS_MAXIMUM) "].\n"
 	"Each additional thread after the first will use an additional 18MiB of RAM,\n"
 	"plus 1.12 MiB of RAM per megabyte of configured read cache size.\n")
 
 cfg(allocation_vdo_bio_rotation_CFG, "vdo_bio_rotation", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BIO_ROTATION, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of I/O operations to enqueue for each bio-submission\n"
-	"thread before directing work to the next. The value must be in range [1..1024].\n")
+	"thread before directing work to the next. The value must be in range ["
+	DM_TO_STRING(DM_VDO_BIO_ROTATION_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_BIO_ROTATION_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_cpu_threads_CFG, "vdo_cpu_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CPU_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for CPU-intensive work such as\n"
-	"hashing or compression for VDO volume. The value must be in range [1..100]\n")
+	"hashing or compression for VDO volume. The value must be in range ["
+	DM_TO_STRING(DM_VDO_CPU_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_CPU_THREADS_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_hash_zone_threads_CFG, "vdo_hash_zone_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_HASH_ZONE_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads across which to subdivide parts of the VDO\n"
 	"processing based on the hash value computed from the block data.\n"
-	"The value must be at in range [0..100].\n"
+	"The value must be at in range [" DM_TO_STRING(DM_VDO_HASH_ZONE_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_HASH_ZONE_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -777,7 +788,8 @@ cfg(allocation_vdo_logical_threads_CFG, "vdo_logical_threads", allocation_CFG_SE
 	"processing based on the hash value computed from the block data.\n"
 	"A logical thread count of 9 or more will require explicitly specifying\n"
 	"a sufficiently large block map cache size, as well.\n"
-	"The value must be in range [0..100].\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_LOGICAL_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_LOGICAL_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -785,7 +797,8 @@ cfg(allocation_vdo_physical_threads_CFG, "vdo_physical_threads", allocation_CFG_
 	"Specifies the number of threads across which to subdivide parts of the VDO\n"
 	"processing based on physical block addresses.\n"
 	"Each additional thread after the first will use an additional 10MiB of RAM.\n"
-	"The value must be in range [0..16].\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_PHYSICAL_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_PHYSICAL_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -811,7 +824,7 @@ cfg(allocation_vdo_max_discard_CFG, "vdo_max_discard", allocation_CFG_SECTION, C
 	"The default and minimum is 1. The maximum is UINT_MAX / 4096.\n")
 
 cfg(allocation_vdo_pool_header_size_CFG, "vdo_pool_header_size", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_POOL_HEADER_SIZE_KB, vsn(2, 3, 12), NULL, 0, NULL,
-	"Specified the emptry header size in KiB at the front and end of vdo pool device.\n")
+	"Specified the empty header size in KiB at the front and end of vdo pool device.\n")
 
 cfg(log_report_command_log_CFG, "report_command_log", log_CFG_SECTION, CFG_PROFILABLE | CFG_DEFAULT_COMMENTED | CFG_DISALLOW_INTERACTIVE, CFG_TYPE_BOOL, DEFAULT_COMMAND_LOG_REPORT, vsn(2, 2, 158), NULL, 0, NULL,
 	"Enable or disable LVM log reporting.\n"
@@ -934,7 +947,7 @@ cfg(backup_archive_CFG, "archive", backup_CFG_SECTION, CFG_DEFAULT_COMMENTED, CF
 	"Think very hard before turning this off.\n")
 
 cfg_runtime(backup_archive_dir_CFG, "archive_dir", backup_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_STRING, vsn(1, 0, 0), 0, NULL,
-	"Location of the metdata archive files.\n"
+	"Location of the metadata archive files.\n"
 	"Remember to back up this directory regularly!\n")
 
 cfg(backup_retain_min_CFG, "retain_min", backup_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_ARCHIVE_NUMBER, vsn(1, 0, 0), NULL, 0, NULL,
@@ -1119,16 +1132,11 @@ cfg(global_lvdisplay_shows_full_device_path_CFG, "lvdisplay_shows_full_device_pa
 	"was never a valid path in the /dev filesystem.\n")
 
 cfg(global_event_activation_CFG, "event_activation", global_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, 1, vsn(2, 3, 1), 0, 0, NULL,
-	"Activate LVs based on system-generated device events.\n"
-	"When a PV appears on the system, a system-generated uevent triggers\n"
-	"the lvm2-pvscan service which runs the pvscan --cache -aay command.\n"
-	"If the new PV completes a VG, pvscan autoactivates LVs in the VG.\n"
-	"When event_activation is disabled, the lvm2-activation services are\n"
-	"generated and run at fixed points during system startup.  These\n"
-	"services run vgchange -aay to autoactivate LVs in VGs that happen\n"
-	"to be present at that point in time.\n"
-	"See the --setautoactivation option or the auto_activation_volume_list\n"
-	"setting to configure autoactivation for specific VGs or LVs.\n")
+	"Disable event based autoactivation commands.\n"
+	"WARNING: setting this to zero may cause machine startup to fail.\n"
+	"Previously, setting this to zero would enable static autoactivation\n"
+	"services (via the lvm2-activation-generator), but the autoactivation\n"
+	"services and generator have been removed.\n")
 
 cfg(global_use_lvmetad_CFG, "use_lvmetad", global_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, 0, vsn(2, 2, 93), 0, vsn(2, 3, 0), NULL,
 	NULL)
@@ -1374,11 +1382,11 @@ cfg(activation_use_linear_target_CFG, "use_linear_target", activation_CFG_SECTIO
 
 cfg(activation_reserved_stack_CFG, "reserved_stack", activation_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_RESERVED_STACK, vsn(1, 0, 0), NULL, 0, NULL,
 	"Stack size in KiB to reserve for use while devices are suspended.\n"
-	"Insufficent reserve risks I/O deadlock during device suspension.\n")
+	"Insufficient reserve risks I/O deadlock during device suspension.\n")
 
 cfg(activation_reserved_memory_CFG, "reserved_memory", activation_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_RESERVED_MEMORY, vsn(1, 0, 0), NULL, 0, NULL,
 	"Memory size in KiB to reserve for use while devices are suspended.\n"
-	"Insufficent reserve risks I/O deadlock during device suspension.\n")
+	"Insufficient reserve risks I/O deadlock during device suspension.\n")
 
 cfg(activation_process_priority_CFG, "process_priority", activation_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_PROCESS_PRIORITY, vsn(1, 0, 0), NULL, 0, NULL,
 	"Nice value used while devices are suspended.\n"
@@ -1504,7 +1512,7 @@ cfg(activation_raid_fault_policy_CFG, "raid_fault_policy", activation_CFG_SECTIO
 	"This includes LVs that have the following segment types:\n"
 	"raid1, raid4, raid5*, and raid6*.\n"
 	"If a device in the LV fails, the policy determines the steps\n"
-	"performed by dmeventd automatically, and the steps perfomed by the\n"
+	"performed by dmeventd automatically, and the steps performed by the\n"
 	"manual command lvconvert --repair --use-policies.\n"
 	"Automatic handling requires dmeventd to be monitoring the LV.\n"
 	"#\n"
@@ -1526,7 +1534,7 @@ cfg_runtime(activation_mirror_image_fault_policy_CFG, "mirror_image_fault_policy
 	"(copies) and a mirror log. A disk log ensures that a mirror LV does\n"
 	"not need to be re-synced (all copies made the same) every time a\n"
 	"machine reboots or crashes. If a device in the LV fails, this policy\n"
-	"determines the steps perfomed by dmeventd automatically, and the steps\n"
+	"determines the steps performed by dmeventd automatically, and the steps\n"
 	"performed by the manual command lvconvert --repair --use-policies.\n"
 	"Automatic handling requires dmeventd to be monitoring the LV.\n"
 	"#\n"
@@ -1801,7 +1809,14 @@ cfg(report_output_format_CFG, "output_format", report_CFG_SECTION, CFG_PROFILABL
 	"    one report per command, each report is prefixed with report's\n"
 	"    name for identification.\n"
 	"  json\n"
-	"    JSON format.\n")
+	"    JSON format.\n"
+	"  json_std\n"
+	"    JSON format that is more compliant with JSON standard.\n"
+        "    Compared to original \"json\" format:\n"
+        "      - it does not use double quotes around numeric values,\n"
+        "      - it uses 'null' for undefined numeric values,\n"
+        "      - it prints string list as proper JSON array of strings instead of a single string."
+	"\n")
 
 cfg(report_compact_output_CFG, "compact_output", report_CFG_SECTION, CFG_PROFILABLE | CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_REP_COMPACT_OUTPUT, vsn(2, 2, 115), NULL, 0, NULL,
 	"Do not print empty values for all report fields.\n"

lib/config/defaults.h

@@ -328,4 +328,10 @@
 
 #define DEFAULT_WWIDS_FILE "/etc/multipath/wwids"
 
+#define PVS_ONLINE_DIR DEFAULT_RUN_DIR "/pvs_online"
+#define VGS_ONLINE_DIR DEFAULT_RUN_DIR "/vgs_online"
+#define PVS_LOOKUP_DIR DEFAULT_RUN_DIR "/pvs_lookup"
+
+#define DEFAULT_DEVICE_ID_SYSFS_DIR "/sys/"  /* trailing / to match dm_sysfs_dir() */
+
 #endif				/* _LVM_DEFAULTS_H */

lib/device/dev-cache.c

@@ -53,6 +53,7 @@ static struct {
 	const char *dev_dir;
 
 	int has_scanned;
+	long st_dev;
 	struct dm_list dirs;
 	struct dm_list files;
 
@@ -79,6 +80,7 @@ static void _dev_init(struct device *dev)
 
 	dm_list_init(&dev->aliases);
 	dm_list_init(&dev->ids);
+	dm_list_init(&dev->wwids);
 }
 
 void dev_destroy_file(struct device *dev)
@@ -350,7 +352,7 @@ static int _add_alias(struct device *dev, const char *path, enum add_hash hash)
 			goto out;
 		}
 
-	if (!(path = dm_pool_strdup(_cache.mem, path)) ||
+	if (!(path = _strdup(path)) ||
 	    !(sl = _zalloc(sizeof(*sl)))) {
 		log_error("Failed to add allias to dev cache.");
 		return 0;
@@ -382,6 +384,22 @@ out:
 	return 1;
 }
 
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen)
+{
+	int ret;
+	int fd;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return 0;
+	ret = read(fd, buf, buf_size);
+	close(fd);
+	if (ret <= 0)
+		return 0;
+	*retlen = ret;
+	return 1;
+}
+
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value)
 {
 	FILE *fp;
@@ -1064,11 +1082,18 @@ static void _insert_dirs(struct dm_list *dirs)
 	struct dir_list *dl;
 	struct udev *udev = NULL;
 	int with_udev;
+	struct stat tinfo;
 
 	with_udev = obtain_device_list_from_udev() &&
 		    (udev = udev_get_library_context());
 
 	dm_list_iterate_items(dl, &_cache.dirs) {
+		if (stat(dl->dir, &tinfo) < 0) {
+			log_warn("WARNING: Cannot use dir %s, %s.",
+				 dl->dir, strerror(errno));
+			continue;
+		}
+		_cache.st_dev = tinfo.st_dev;
 		if (with_udev) {
 			if (!_insert_udev_dir(udev, dl->dir))
 				log_debug_devs("%s: Failed to insert devices from "
@@ -1091,9 +1116,17 @@ static int _device_in_udev_db(const dev_t d)
 static void _insert_dirs(struct dm_list *dirs)
 {
 	struct dir_list *dl;
+	struct stat tinfo;
 
-	dm_list_iterate_items(dl, &_cache.dirs)
+	dm_list_iterate_items(dl, &_cache.dirs) {
+		if (stat(dl->dir, &tinfo) < 0) {
+			log_warn("WARNING: Cannot use dir %s, %s.",
+				 dl->dir, strerror(errno));
+			continue;
+		}
+		_cache.st_dev = tinfo.st_dev;
 		_insert_dir(dl->dir);
+	}
 }
 
 #endif	/* UDEV_SYNC_SUPPORT */
@@ -1128,6 +1161,11 @@ static int _insert(const char *path, const struct stat *info,
 			return 1;
 		}
 
+		if (info->st_dev != _cache.st_dev) {
+			log_debug_devs("%s: Different filesystem in directory", path);
+			return 1;
+		}
+
 		if (rec && !_insert_dir(path))
 			return 0;
 	} else {		/* add a device */
@@ -1141,6 +1179,17 @@ static int _insert(const char *path, const struct stat *info,
 	return 1;
 }
 
+static void _drop_all_aliases(struct device *dev)
+{
+	struct dm_str_list *strl, *strl2;
+
+	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
+		log_debug("Drop alias for %d:%d %s.", (int)MAJOR(dev->dev), (int)MINOR(dev->dev), strl->str);
+		dm_hash_remove(_cache.names, strl->str);
+		dm_list_del(&strl->list);
+	}
+}
+
 void dev_cache_scan(struct cmd_context *cmd)
 {
 	log_debug_devs("Creating list of system devices.");
@@ -1225,7 +1274,7 @@ int dev_cache_init(struct cmd_context *cmd)
 	if (!(_cache.mem = dm_pool_create("dev_cache", 10 * 1024)))
 		return_0;
 
-	if (!(_cache.names = dm_hash_create(120)) ||
+	if (!(_cache.names = dm_hash_create(1020)) ||
 	    !(_cache.vgid_index = dm_hash_create(30)) ||
 	    !(_cache.lvid_index = dm_hash_create(29))) {
 		dm_pool_destroy(_cache.mem);
@@ -1304,6 +1353,7 @@ int dev_cache_exit(void)
 		dm_hash_iterate(n, _cache.names) {
 			dev = (struct device *) dm_hash_get_data(_cache.names, n);
 			free_dids(&dev->ids);
+			free_wwids(&dev->wwids);
 		}
 	}
 
@@ -1350,59 +1400,6 @@ int dev_cache_add_dir(const char *path)
 	return 1;
 }
 
-/* Check cached device name is still valid before returning it */
-/* This should be a rare occurrence */
-/* set quiet if the cache is expected to be out-of-date */
-/* FIXME Make rest of code pass/cache struct device instead of dev_name */
-const char *dev_name_confirmed(struct device *dev, int quiet)
-{
-	struct stat buf;
-	const char *name;
-	int r;
-
-	if ((dev->flags & DEV_REGULAR))
-		return dev_name(dev);
-
-	while ((r = stat(name = dm_list_item(dev->aliases.n,
-					  struct dm_str_list)->str, &buf)) ||
-	       (buf.st_rdev != dev->dev)) {
-		if (r < 0) {
-			if (quiet)
-				log_sys_debug("stat", name);
-			else
-				log_sys_error("stat", name);
-		}
-		if (quiet)
-			log_debug_devs("Path %s no longer valid for device(%d,%d)",
-				       name, (int) MAJOR(dev->dev),
-				       (int) MINOR(dev->dev));
-		else
-			log_warn("Path %s no longer valid for device(%d,%d)",
-				 name, (int) MAJOR(dev->dev),
-				 (int) MINOR(dev->dev));
-
-		/* Remove the incorrect hash entry */
-		dm_hash_remove(_cache.names, name);
-
-		/* Leave list alone if there isn't an alternative name */
-		/* so dev_name will always find something to return. */
-		/* Otherwise add the name to the correct device. */
-		if (dm_list_size(&dev->aliases) > 1) {
-			dm_list_del(dev->aliases.n);
-			if (!r)
-				_insert(name, &buf, 0, obtain_device_list_from_udev());
-			continue;
-		}
-
-		/* Scanning issues this inappropriately sometimes. */
-		log_debug_devs("Aborting - please provide new pathname for what "
-			       "used to be %s", name);
-		return NULL;
-	}
-
-	return dev_name(dev);
-}
-
 struct device *dev_hash_get(const char *name)
 {
 	return (struct device *) dm_hash_lookup(_cache.names, name);
@@ -1431,26 +1428,23 @@ static void _remove_alias(struct device *dev, const char *name)
  * deactivated LV.  Those old paths are all invalid and are dropped here.
  */
 
-static void _verify_aliases(struct device *dev, const char *newname)
+void dev_cache_verify_aliases(struct device *dev)
 {
 	struct dm_str_list *strl, *strl2;
 	struct stat st;
 
 	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
-		/* newname was just stat'd and added by caller */
-		if (newname && !strcmp(strl->str, newname))
-			continue;
-
 		if (stat(strl->str, &st) || (st.st_rdev != dev->dev)) {
-			log_debug("Drop invalid path %s for %d:%d (new path %s).",
-				  strl->str, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), newname ?: "");
+			log_debug("Drop alias for %d:%d invalid path %s %d:%d.",
+				  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), strl->str,
+				  (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev));
 			dm_hash_remove(_cache.names, strl->str);
 			dm_list_del(&strl->list);
 		}
 	}
 }
 
-struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f)
+static struct device *_dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f, int existing)
 {
 	struct device *dev = (struct device *) dm_hash_lookup(_cache.names, name);
 	struct stat st;
@@ -1464,13 +1458,18 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	if (dev && (dev->flags & DEV_REGULAR))
 		return dev;
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
 	/*
-	 * The requested path is invalid, remove any dev-cache
-	 * info for it.
+	 * The requested path is invalid, remove any dev-cache info for it.
 	 */
 	if (stat(name, &st)) {
 		if (dev) {
-			log_print("Device path %s is invalid for %d:%d %s.",
+			log_debug("Device path %s is invalid for %d:%d %s.",
 				  name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev));
 
 			dm_hash_remove(_cache.names, name);
@@ -1478,11 +1477,17 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 			_remove_alias(dev, name);
 
 			/* Remove any other names in dev->aliases that are incorrect. */
-			_verify_aliases(dev, NULL);
+			dev_cache_verify_aliases(dev);
 		}
 		return NULL;
 	}
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
 	if (!S_ISBLK(st.st_mode)) {
 		log_debug("Not a block device %s.", name);
 		return NULL;
@@ -1493,26 +1498,41 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	 * Remove incorrect info and then add new dev-cache entry.
 	 */
 	if (dev && (st.st_rdev != dev->dev)) {
-		log_debug("Device path %s does not match %d:%d %s.",
-			  name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev));
+		struct device *dev_by_devt = (struct device *) btree_lookup(_cache.devices, (uint32_t) st.st_rdev);
 
-		dm_hash_remove(_cache.names, name);
+		/*
+		 * lvm commands create this condition when they
+		 * activate/deactivate LVs combined with creating new LVs.
+		 * The command does not purge dev structs when deactivating
+		 * an LV (which it probably should do), but the better
+		 * approach would be not using dev-cache at all for LVs.
+		 */
 
-		_remove_alias(dev, name);
+		log_debug("Dropping aliases for device entry %d:%d %s for new device %d:%d %s.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), dev_name(dev),
+			  (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev), name);
 
-		/* Remove any other names in dev->aliases that are incorrect. */
-		_verify_aliases(dev, NULL);
+		_drop_all_aliases(dev);
 
-		/* Add new dev-cache entry next. */
-		dev = NULL;
-	}
+		if (dev_by_devt) {
+			log_debug("Dropping aliases for device entry %d:%d %s for new device %d:%d %s.",
+				   (int)MAJOR(dev_by_devt->dev), (int)MINOR(dev_by_devt->dev), dev_name(dev_by_devt),
+				   (int)MAJOR(st.st_rdev), (int)MINOR(st.st_rdev), name);
+
+			_drop_all_aliases(dev_by_devt);
+		}
+
+#if 0
+		/*
+		 * I think only lvm's own dm devs should be added here, so use
+		 * a warning to look for any other unknown cases.
+		 */
+		if (MAJOR(st.st_rdev) != cmd->dev_types->device_mapper_major) {
+			log_warn("WARNING: new device appeared %d:%d %s",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+		}
+#endif
 
-	/*
-	 * Either add a new struct dev for st_rdev and name,
-	 * or add name as a new alias for an existing struct dev
-	 * for st_rdev.
-	 */
-	if (!dev) {
 		if (!_insert_dev(name, st.st_rdev))
 			return_NULL;
 
@@ -1524,9 +1544,77 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 			return NULL;
 		}
 
-		_verify_aliases(dev, name);
+		goto out;
 	}
 
+	if (dev && dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_warn("Ignoring dev with no valid paths for %s.", name);
+		return NULL;
+	}
+
+	if (!dev && existing)
+		return_NULL;
+
+	/*
+	 * This case should never be hit for a PV. It should only
+	 * happen when the command is opening a new LV it has created.
+	 * Add an arg to all callers indicating when the arg should be
+	 * new (for an LV) and not existing.
+	 * FIXME: fix this further by not using dev-cache struct devs
+	 * at all for new dm devs (LVs) that lvm uses.  Make the
+	 * dev-cache contain only devs for PVs.
+	 * Places to fix that use a dev for LVs include:
+	 * . lv_resize opening lv to discard
+	 * . wipe_lv opening lv to zero it
+	 * . _extend_sanlock_lv opening lv to extend it
+	 * . _write_log_header opening lv to write header
+	 * Also, io to LVs should not go through bcache.
+	 * bcache should contain only labels and metadata
+	 * scanned from PVs.
+	 */
+	if (!dev) {
+		/*
+		 * This case should only be used for new devices created by this
+		 * command (opening LVs it's created), so if a dev exists for the
+		 * dev_t referenced by the name, then drop all aliases for before
+		 * _insert_dev adds the new name.  lvm commands actually hit this
+		 * fairly often when it uses some LV, deactivates the LV, then
+		 * creates some new LV which ends up with the same major:minor.
+		 * Without dropping the aliases, it's plausible that lvm commands
+		 * could end up using the wrong dm device.
+		 */
+		struct device *dev_by_devt = (struct device *) btree_lookup(_cache.devices, (uint32_t) st.st_rdev);
+		if (dev_by_devt) {
+			log_debug("Dropping aliases for %d:%d before adding new path %s.",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+			_drop_all_aliases(dev_by_devt);
+		}
+
+#if 0
+		/*
+		 * I think only lvm's own dm devs should be added here, so use
+		 * a warning to look for any other unknown cases.
+		 */
+		if (MAJOR(st.st_rdev) != cmd->dev_types->device_mapper_major) {
+			log_warn("WARNING: new device appeared %d:%d %s",
+				  (int)MAJOR(st.st_rdev), (int)(MINOR(st.st_rdev)), name);
+		}
+#endif
+
+		if (!_insert_dev(name, st.st_rdev))
+			return_NULL;
+
+		/* Get the struct dev that was just added. */
+		dev = (struct device *) dm_hash_lookup(_cache.names, name);
+
+		if (!dev) {
+			log_error("Failed to get device %s", name);
+			return NULL;
+		}
+	}
+
+ out:
 	/*
 	 * The caller passed a filter if they only want the dev if it
 	 * passes filters.
@@ -1536,18 +1624,6 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 		return dev;
 
 	ret = f->passes_filter(cmd, f, dev, NULL);
-
-	/*
-	 * This might happen if this function is called before
-	 * filters can do i/o.  I don't think this will happen
-	 * any longer and this EAGAIN case can be removed.
-	 */
-	if (ret == -EAGAIN) {
-		log_debug_devs("dev_cache_get filter deferred %s", dev_name(dev));
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		ret = 1;
-	}
-
 	if (!ret) {
 		log_debug_devs("dev_cache_get filter excludes %s", dev_name(dev));
 		return NULL;
@@ -1556,63 +1632,23 @@ struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct d
 	return dev;
 }
 
-struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t dev, struct dev_filter *f, int *filtered)
+struct device *dev_cache_get_existing(struct cmd_context *cmd, const char *name, struct dev_filter *f)
 {
-	char path[PATH_MAX];
-	const char *sysfs_dir;
-	struct stat info;
-	struct device *d = (struct device *) btree_lookup(_cache.devices, (uint32_t) dev);
-	int ret;
+	return _dev_cache_get(cmd, name, f, 1);
+}
 
-	if (filtered)
-		*filtered = 0;
+struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f)
+{
+	return _dev_cache_get(cmd, name, f, 0);
+}
 
-	if (!d) {
-		sysfs_dir = dm_sysfs_dir();
-		if (sysfs_dir && *sysfs_dir) {
-			/* First check if dev is sysfs to avoid useless scan */
-			if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d",
-					sysfs_dir, (int)MAJOR(dev), (int)MINOR(dev)) < 0) {
-				log_error("dm_snprintf partition failed.");
-				return NULL;
-			}
-
-			if (lstat(path, &info)) {
-				log_debug("No sysfs entry for %d:%d errno %d at %s.",
-					  (int)MAJOR(dev), (int)MINOR(dev), errno, path);
-				return NULL;
-			}
-		}
-
-		log_debug_devs("Device num not found in dev_cache repeat dev_cache_scan for %d:%d",
-				(int)MAJOR(dev), (int)MINOR(dev));
-		dev_cache_scan(cmd);
-		d = (struct device *) btree_lookup(_cache.devices, (uint32_t) dev);
-
-		if (!d)
-			return NULL;
-	}
-
-	if (d->flags & DEV_REGULAR)
-		return d;
-
-	if (!f)
-		return d;
-
-	ret = f->passes_filter(cmd, f, d, NULL);
-
-	if (ret == -EAGAIN) {
-		log_debug_devs("get device by number defer filter %s", dev_name(d));
-		d->flags |= DEV_FILTER_AFTER_SCAN;
-		ret = 1;
-	}
-
-	if (ret)
-		return d;
-
-	if (filtered)
-		*filtered = 1;
+struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t devt)
+{
+	struct device *dev = (struct device *) btree_lookup(_cache.devices, (uint32_t) devt);
 
+	if (dev)
+		return dev;
+	log_debug_devs("No devno %d:%d in dev cache.", (int)MAJOR(devt), (int)MINOR(devt));
 	return NULL;
 }
 
@@ -1658,16 +1694,9 @@ struct device *dev_iter_get(struct cmd_context *cmd, struct dev_iter *iter)
 
 		f = iter->filter;
 
-		if (f && !(d->flags & DEV_REGULAR)) {
+		if (f && !(d->flags & DEV_REGULAR))
 			ret = f->passes_filter(cmd, f, d, NULL);
 
-			if (ret == -EAGAIN) {
-				log_debug_devs("get device by iter defer filter %s", dev_name(d));
-				d->flags |= DEV_FILTER_AFTER_SCAN;
-				ret = 1;
-			}
-		}
-
 		if (!f || (d->flags & DEV_REGULAR) || ret)
 			return d;
 	}
@@ -1682,8 +1711,10 @@ int dev_fd(struct device *dev)
 
 const char *dev_name(const struct device *dev)
 {
-	return (dev && dev->aliases.n) ? dm_list_item(dev->aliases.n, struct dm_str_list)->str :
-	    unknown_device_name();
+	if (dev && dev->aliases.n && !dm_list_empty(&dev->aliases))
+		return dm_list_item(dev->aliases.n, struct dm_str_list)->str;
+	else
+		return unknown_device_name();
 }
 
 bool dev_cache_has_md_with_end_superblock(struct dev_types *dt)
@@ -1831,7 +1862,7 @@ int setup_devices_file(struct cmd_context *cmd)
  * Add all system devices to dev-cache, and attempt to
  * match all devices_file entries to dev-cache entries.
  */
-static int _setup_devices(struct cmd_context *cmd, int no_file_match)
+int setup_devices(struct cmd_context *cmd)
 {
 	int file_exists;
 	int lock_mode = 0;
@@ -1850,6 +1881,15 @@ static int _setup_devices(struct cmd_context *cmd, int no_file_match)
 
 	file_exists = devices_file_exists(cmd);
 
+	/*
+	 * Fail if user specifies a file name that doesn't exist and
+	 * the command is not creating a new devices file.
+	 */
+	if (!file_exists && !cmd->create_edit_devices_file && cmd->devicesfile && strlen(cmd->devicesfile)) {
+		log_error("Devices file not found: %s", cmd->devices_file_path);
+		return 0;
+	}
+
 	/*
 	 * Removing the devices file is another way of disabling the use of
 	 * a devices file, unless the command creates the devices file.
@@ -1897,10 +1937,9 @@ static int _setup_devices(struct cmd_context *cmd, int no_file_match)
 
 	if (!file_exists) {
 		/*
-		 * pvcreate/vgcreate/vgimportdevices/lvmdevices-add create
-		 * a new devices file here if it doesn't exist.
-		 * They have the create_edit_devices_file flag set.
-		 * First they create/lock-ex the devices file lockfile.
+		 * pvcreate/vgcreate create a new devices file here if it
+		 * doesn't exist.  They have create_edit_devices_file=1.
+		 * First create/lock-ex the devices file lockfile.
 		 * Other commands will not use a devices file if none exists.
 		 */
 		lock_mode = LOCK_EX;
@@ -1958,13 +1997,6 @@ static int _setup_devices(struct cmd_context *cmd, int no_file_match)
 	 */
 	dev_cache_scan(cmd);
 
-	/*
-	 * The caller uses "no_file_match" if it wants to match specific devs
-	 * itself, instead of matching everything in device_ids_match.
-	 */
-	if (no_file_match && cmd->enable_devices_file)
-		return 1;
-
 	/*
 	 * Match entries from cmd->use_devices with device structs in dev-cache.
 	 */
@@ -1973,16 +2005,6 @@ static int _setup_devices(struct cmd_context *cmd, int no_file_match)
 	return 1;
 }
 
-int setup_devices(struct cmd_context *cmd)
-{
-	return _setup_devices(cmd, 0);
-}
-
-int setup_devices_no_file_match(struct cmd_context *cmd)
-{
-	return _setup_devices(cmd, 1);
-}
-
 /*
  * The alternative to setup_devices() when the command is interested
  * in using only one PV.
@@ -2051,3 +2073,244 @@ int setup_device(struct cmd_context *cmd, const char *devname)
 	return 1;
 }
 
+/*
+ * autoactivation is specialized/optimized to look only at command args,
+ * so this just sets up the devices file, then individual devices are
+ * added to dev-cache and matched with device_ids.
+ */
+
+int setup_devices_for_online_autoactivation(struct cmd_context *cmd)
+{
+	if (cmd->enable_devices_list) {
+		if (!_setup_devices_list(cmd))
+			return_0;
+		return 1;
+	}
+
+	if (!setup_devices_file(cmd))
+		return_0;
+
+	if (!cmd->enable_devices_file)
+		return 1;
+
+	if (!devices_file_exists(cmd)) {
+		log_debug("Devices file not found, ignoring.");
+		cmd->enable_devices_file = 0;
+		return 1;
+	}
+
+	if (!lock_devices_file(cmd, LOCK_SH)) {
+		log_error("Failed to lock the devices file to read.");
+		return 0;
+	}
+
+	if (!device_ids_read(cmd)) {
+		log_error("Failed to read the devices file.");
+		unlock_devices_file(cmd);
+		return 0;
+	}
+
+	unlock_devices_file(cmd);
+	return 1;
+}
+
+
+/* Get a device name from a devno. */
+
+static char *_get_devname_from_devno(struct cmd_context *cmd, dev_t devno)
+{
+	char path[PATH_MAX];
+	char devname[PATH_MAX] = { 0 };
+	char namebuf[NAME_LEN];
+	char line[1024];
+	int major = MAJOR(devno);
+	int minor = MINOR(devno);
+	int line_major;
+	int line_minor;
+	uint64_t line_blocks;
+	DIR *dir;
+	struct dirent *dirent;
+	FILE *fp;
+
+	if (!devno)
+		return NULL;
+
+	/*
+	 * $ ls /sys/dev/block/8:0/device/block/
+	 * sda
+	 */
+	if (major_is_scsi_device(cmd->dev_types, major)) {
+		if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/device/block",
+				dm_sysfs_dir(), major, minor) < 0) {
+			return NULL;
+		}
+
+		if (!(dir = opendir(path)))
+			goto try_partition;
+
+		while ((dirent = readdir(dir))) {
+			if (dirent->d_name[0] == '.')
+				continue;
+			if (dm_snprintf(devname, sizeof(devname), "/dev/%s", dirent->d_name) < 0) {
+				devname[0] = '\0';
+				stack;
+			}
+			break;
+		}
+		closedir(dir);
+
+		if (devname[0]) {
+			log_debug("Found %s for %d:%d from sys", devname, major, minor);
+			return _strdup(devname);
+		}
+		return NULL;
+	}
+
+	/*
+	 * $ cat /sys/dev/block/253:3/dm/name
+	 * mpatha
+	 */
+	if (major == cmd->dev_types->device_mapper_major) {
+		if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/dm/name",
+				dm_sysfs_dir(), major, minor) < 0) {
+			return NULL;
+		}
+
+		if (!get_sysfs_value(path, namebuf, sizeof(namebuf), 0))
+			return NULL;
+
+		if (dm_snprintf(devname, sizeof(devname), "/dev/mapper/%s", namebuf) < 0) {
+			devname[0] = '\0';
+			stack;
+		}
+
+		if (devname[0]) {
+			log_debug("Found %s for %d:%d from sys dm", devname, major, minor);
+			return _strdup(devname);
+		}
+		return NULL;
+	}
+
+	/*
+	 * /proc/partitions lists
+	 * major minor #blocks name
+	 */
+
+try_partition:
+	if (!(fp = fopen("/proc/partitions", "r")))
+		return NULL;
+
+	while (fgets(line, sizeof(line), fp)) {
+		if (sscanf(line, "%u %u %llu %s", &line_major, &line_minor, (unsigned long long *)&line_blocks, namebuf) != 4)
+			continue;
+		if (line_major != major)
+			continue;
+		if (line_minor != minor)
+			continue;
+
+		if (dm_snprintf(devname, sizeof(devname), "/dev/%s", namebuf) < 0) {
+			devname[0] = '\0';
+			stack;
+		}
+		break;
+	}
+	fclose(fp);
+
+	if (devname[0]) {
+		log_debug("Found %s for %d:%d from proc", devname, major, minor);
+		return _strdup(devname);
+	}
+
+	/*
+	 * If necessary, this could continue searching by stat'ing /dev entries.
+	 */
+
+	return NULL;
+}
+
+int setup_devname_in_dev_cache(struct cmd_context *cmd, const char *devname)
+{
+	struct stat buf;
+	struct device *dev;
+
+	if (stat(devname, &buf) < 0) {
+		log_error("Cannot access device %s.", devname);
+		return 0;
+	}
+
+	if (!S_ISBLK(buf.st_mode)) {
+		log_error("Invaild device type %s.", devname);
+		return 0;
+	}
+
+	if (!_insert_dev(devname, buf.st_rdev))
+		return_0;
+
+	if (!(dev = (struct device *) dm_hash_lookup(_cache.names, devname)))
+		return_0;
+
+	return 1;
+}
+
+int setup_devno_in_dev_cache(struct cmd_context *cmd, dev_t devno)
+{
+	const char *devname;
+
+	if (!(devname = _get_devname_from_devno(cmd, devno)))
+		return_0;
+
+	return setup_devname_in_dev_cache(cmd, devname);
+}
+
+struct device *setup_dev_in_dev_cache(struct cmd_context *cmd, dev_t devno, const char *devname)
+{
+	struct device *dev;
+	struct stat buf;
+	int major = (int)MAJOR(devno);
+	int minor = (int)MINOR(devno);
+
+	if (devname) {
+		if (stat(devname, &buf) < 0) {
+			log_error("Cannot access device %s for %d:%d.", devname, major, minor);
+			if (!devno)
+				return_NULL;
+			if (!(devname = _get_devname_from_devno(cmd, devno))) {
+				log_error("No device name found from %d:%d.", major, minor);
+				return_NULL;
+			}
+			if (stat(devname, &buf) < 0) {
+				log_error("Cannot access device %s from %d:%d.", devname, major, minor);
+				return_NULL;
+			}
+		}
+	} else {
+		if (!(devname = _get_devname_from_devno(cmd, devno))) {
+			log_error("No device name found from %d:%d.", major, minor);
+			return_NULL;
+		}
+		if (stat(devname, &buf) < 0) {
+			log_error("Cannot access device %s from %d:%d.", devname, major, minor);
+			return_NULL;
+		}
+	}
+
+	if (!S_ISBLK(buf.st_mode)) {
+		log_error("Invaild device type %s.", devname);
+		return_NULL;
+	}
+
+	if (devno && (buf.st_rdev != devno)) {
+		log_warn("Found %s devno %d:%d expected %d:%d.", devname,
+			  (int)MAJOR(buf.st_rdev), (int)MINOR(buf.st_rdev), major, minor);
+	}
+
+	if (!_insert_dev(devname, buf.st_rdev))
+		return_NULL;
+
+	if (!(dev = (struct device *) dm_hash_lookup(_cache.names, devname))) {
+		log_error("Device lookup failed for %d:%d %s", major, minor, devname);
+		return_NULL;
+	}
+
+	return dev;
+}

lib/device/dev-cache.h

@@ -53,8 +53,9 @@ int dev_cache_has_scanned(void);
 
 int dev_cache_add_dir(const char *path);
 struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f);
-
-struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t device, struct dev_filter *f, int *filtered);
+struct device *dev_cache_get_existing(struct cmd_context *cmd, const char *name, struct dev_filter *f);
+struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t devt);
+void dev_cache_verify_aliases(struct device *dev);
 
 struct device *dev_hash_get(const char *name);
 
@@ -73,11 +74,16 @@ void dev_cache_failed_path(struct device *dev, const char *path);
 bool dev_cache_has_md_with_end_superblock(struct dev_types *dt);
 
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value);
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen);
 int get_dm_uuid_from_sysfs(char *buf, size_t buf_size, int major, int minor);
 
 int setup_devices_file(struct cmd_context *cmd);
 int setup_devices(struct cmd_context *cmd);
-int setup_devices_no_file_match(struct cmd_context *cmd);
 int setup_device(struct cmd_context *cmd, const char *devname);
 
+int setup_devices_for_online_autoactivation(struct cmd_context *cmd);
+int setup_devname_in_dev_cache(struct cmd_context *cmd, const char *devname);
+int setup_devno_in_dev_cache(struct cmd_context *cmd, dev_t devno);
+struct device *setup_dev_in_dev_cache(struct cmd_context *cmd, dev_t devno, const char *devname);
+
 #endif

lib/device/dev-io.c

@@ -58,6 +58,9 @@ static int _dev_get_size_file(struct device *dev, uint64_t *size)
 	const char *name = dev_name(dev);
 	struct stat info;
 
+	if (dm_list_empty(&dev->aliases))
+		return_0;
+
 	if (dev->size_seqno == _dev_size_seqno) {
 		log_very_verbose("%s: using cached size %" PRIu64 " sectors",
 				 name, dev->size);
@@ -87,7 +90,7 @@ static int _dev_get_size_dev(struct device *dev, uint64_t *size)
 	int do_close = 0;
 
 	if (dm_list_empty(&dev->aliases))
-		return 0;
+		return_0;
 
 	if (dev->size_seqno == _dev_size_seqno) {
 		log_very_verbose("%s: using cached size %" PRIu64 " sectors",
@@ -305,6 +308,13 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 	if ((flags & O_EXCL))
 		need_excl = 1;
 
+	if (dm_list_empty(&dev->aliases)) {
+		/* shouldn't happen */
+		log_print("Cannot open device %d:%d with no valid paths.", (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		return 0;
+	}
+	name = dev_name(dev);
+
 	if (dev->fd >= 0) {
 		if (((dev->flags & DEV_OPENED_RW) || !need_rw) &&
 		    ((dev->flags & DEV_OPENED_EXCL) || !need_excl)) {
@@ -314,7 +324,7 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 
 		if (dev->open_count && !need_excl)
 			log_debug_devs("%s: Already opened read-only. Upgrading "
-				       "to read-write.", dev_name(dev));
+				       "to read-write.", name);
 
 		/* dev_close_immediate will decrement this */
 		dev->open_count++;
@@ -327,11 +337,7 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 
 	if (critical_section())
 		/* FIXME Make this log_error */
-		log_verbose("dev_open(%s) called while suspended",
-			    dev_name(dev));
-
-	if (!(name = dev_name_confirmed(dev, quiet)))
-		return_0;
+		log_verbose("dev_open(%s) called while suspended", name);
 
 #ifdef O_DIRECT_SUPPORT
 	if (direct) {
@@ -372,9 +378,9 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 		}
 #endif
 		if (quiet)
-			log_sys_debug("open", name);
+			log_debug("Failed to open device path %s (%d).", name, errno);
 		else
-			log_sys_error("open", name);
+			log_error("Failed to open device path %s (%d).", name, errno);
 
 		dev->flags |= DEV_OPEN_FAILURE;
 		return 0;
@@ -415,10 +421,12 @@ int dev_open_flags(struct device *dev, int flags, int direct, int quiet)
 	if ((flags & O_CREAT) && !(flags & O_TRUNC))
 		dev->end = lseek(dev->fd, (off_t) 0, SEEK_END);
 
-	log_debug_devs("Opened %s %s%s%s", dev_name(dev),
-		       dev->flags & DEV_OPENED_RW ? "RW" : "RO",
-		       dev->flags & DEV_OPENED_EXCL ? " O_EXCL" : "",
-		       dev->flags & DEV_O_DIRECT ? " O_DIRECT" : "");
+	if (!quiet) {
+		log_debug_devs("Opened %s %s%s%s", name,
+				dev->flags & DEV_OPENED_RW ? "RW" : "RO",
+				dev->flags & DEV_OPENED_EXCL ? " O_EXCL" : "",
+				dev->flags & DEV_O_DIRECT ? " O_DIRECT" : "");
+	}
 
 	dev->flags &= ~DEV_OPEN_FAILURE;
 	return 1;

lib/device/dev-luks.c

@@ -23,9 +23,6 @@ int dev_is_luks(struct cmd_context *cmd, struct device *dev, uint64_t *offset_fo
 	char buf[LUKS_SIGNATURE_SIZE];
 	int ret = -1;
 
-	if (!scan_bcache)
-		return -EAGAIN;
-
 	if (offset_found)
 		*offset_found = 0;
 

lib/device/dev-md.c

@@ -178,12 +178,6 @@ static int _dev_is_md_component_native(struct device *dev, uint64_t *offset_foun
 	uint64_t size, sb_offset = 0;
 	int ret;
 
-	/* i/o layer has not been set up */
-	if (!scan_bcache) {
-		log_error(INTERNAL_ERROR "dev_is_md_component_native requires io layer.");
-		return -1;
-	}
-
 	if (!dev_get_size(dev, &size)) {
 		stack;
 		return -1;

lib/device/dev-mpath.c

@@ -17,12 +17,14 @@
 #include "lib/activate/activate.h"
 #include "lib/commands/toolcontext.h"
 #include "lib/device/device_id.h"
+#include "lib/datastruct/str_list.h"
 #ifdef UDEV_SYNC_SUPPORT
 #include <libudev.h>
 #include "lib/device/dev-ext-udev-constants.h"
 #endif
 
 #include <dirent.h>
+#include <ctype.h>
 
 #define MPATH_PREFIX "mpath-"
 
@@ -35,21 +37,175 @@
  * If dm-3 is not an mpath device, then the constant "1" is stored in
  * the hash table with the key of the dm minor number.
  */
-static struct dm_pool *_hash_mem;
+static struct dm_pool *_wwid_mem;
 static struct dm_hash_table *_minor_hash_tab;
 static struct dm_hash_table *_wwid_hash_tab;
+static struct dm_list _ignored;
+static struct dm_list _ignored_exceptions;
 
 #define MAX_WWID_LINE 512
 
-/*
- * do we need to check the multipath.conf blacklist?
- */
+static void _read_blacklist_file(const char *path)
+{
+	FILE *fp;
+	char line[MAX_WWID_LINE];
+	char wwid[MAX_WWID_LINE];
+	char *word, *p;
+	int section_black = 0;
+	int section_exceptions = 0;
+	int found_quote;
+	int found_type;
+	int i, j;
 
-static void _read_wwid_file(const char *config_wwids_file)
+	if (!(fp = fopen(path, "r")))
+		return;
+
+	while (fgets(line, sizeof(line), fp)) {
+		word = NULL;
+
+		/* skip initial white space on the line */
+		for (i = 0; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (isspace(line[i]))
+				continue;
+			word = &line[i];
+			break;
+		}
+
+		if (!word || word[0] == '#')
+			continue;
+
+		/* identify the start of the section we want to read */
+		if (strchr(word, '{')) {
+			if (!strncmp(word, "blacklist_exceptions", 20))
+				section_exceptions = 1;
+			else if (!strncmp(word, "blacklist", 9))
+				section_black = 1;
+			continue;
+		}
+		/* identify the end of the section we've been reading */
+		if (strchr(word, '}')) {
+			section_exceptions = 0;
+			section_black = 0;
+			continue;
+		}
+		/* skip lines that are not in a section we want */
+		if (!section_black && !section_exceptions)
+			continue;
+
+		/*
+		 * read a wwid from the blacklist{_exceptions} section.
+		 * does not recognize other non-wwid entries in the
+		 * section, and skips those (should the entire mp
+		 * config filtering be disabled if non-wwids are seen?
+		 */
+		if (!(p = strstr(word, "wwid")))
+			continue;
+
+		i += 4; /* skip "wwid" */
+
+		/*
+		 * copy wwid value from the line.
+		 * the wwids copied here need to match the
+		 * wwids read from /etc/multipath/wwids,
+		 * which are matched to wwids from sysfs.
+		 */
+
+		memset(wwid, 0, sizeof(wwid));
+		found_quote = 0;
+		found_type = 0;
+		j = 0;
+
+		for (; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (!j && isspace(line[i]))
+				continue;
+			if (isspace(line[i]))
+				break;
+			/* quotes around wwid are optional */
+			if ((line[i] == '"') && !found_quote) {
+				found_quote = 1;
+				continue;
+			}
+			/* second quote is end of wwid */
+			if ((line[i] == '"') && found_quote)
+				break;
+			/* exclude initial 3/2/1 for naa/eui/t10 */
+			if (!j && !found_type &&
+			    ((line[i] == '3') || (line[i] == '2') || (line[i] == '1'))) {
+				found_type = 1;
+				continue;
+			}
+
+			wwid[j] = line[i];
+			j++;
+		}
+
+		if (j < 8)
+			continue;
+
+		log_debug("multipath wwid %s in %s %s",
+			  wwid, section_exceptions ? "blacklist_exceptions" : "blacklist", path);
+
+		if (section_exceptions) {
+			if (!str_list_add(_wwid_mem, &_ignored_exceptions, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		} else {
+			if (!str_list_add(_wwid_mem, &_ignored, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		}
+	}
+
+	if (fclose(fp))
+		stack;
+}
+
+static void _read_wwid_exclusions(void)
+{
+	char path[PATH_MAX] = { 0 };
+	DIR *dir;
+	struct dirent *de;
+	struct dm_str_list *sl, *sl2;
+	int rem_count = 0;
+
+	_read_blacklist_file("/etc/multipath.conf");
+
+	if ((dir = opendir("/etc/multipath/conf.d"))) {
+		while ((de = readdir(dir))) {
+			if (de->d_name[0] == '.')
+				continue;
+			snprintf(path, PATH_MAX-1, "/etc/multipath/conf.d/%s", de->d_name);
+			_read_blacklist_file(path);
+		}
+		closedir(dir);
+	}
+
+	/* for each wwid in ignored_exceptions, remove it from ignored */
+
+	dm_list_iterate_items_safe(sl, sl2, &_ignored) {
+		if (str_list_match_item(&_ignored_exceptions, sl->str))
+			str_list_del(&_ignored, sl->str);
+	}
+
+	/* for each wwid in ignored, remove it from wwid_hash */
+
+	dm_list_iterate_items(sl, &_ignored) {
+		dm_hash_remove_binary(_wwid_hash_tab, sl->str, strlen(sl->str));
+		rem_count++;
+	}
+
+	if (rem_count)
+		log_debug("multipath config ignored %d wwids", rem_count);
+}
+
+static void _read_wwid_file(const char *config_wwids_file, int *entries)
 {
 	FILE *fp;
 	char line[MAX_WWID_LINE];
 	char *wwid, *p;
+	char typestr[2] = { 0 };
 	int count = 0;
 
 	if (config_wwids_file[0] != '/') {
@@ -71,8 +227,17 @@ static void _read_wwid_file(const char *config_wwids_file)
 		if (line[0] == '/')
 			wwid++;
 
-		/* skip the initial '3' */
-		wwid++;
+
+		/*
+		 * the initial character is the id type,
+		 * 1 is t10, 2 is eui, 3 is naa, 8 is scsi name.
+		 * wwids are stored in the hash table without the type charater.
+		 * It seems that sometimes multipath does not include
+		 * the type charater (seen with t10 scsi_debug devs).
+		 */
+		typestr[0] = *wwid;
+		if (typestr[0] == '1' || typestr[0] == '2' || typestr[0] == '3')
+			wwid++;
 
 		if ((p = strchr(wwid, '/')))
 			*p = '\0';
@@ -85,6 +250,7 @@ static void _read_wwid_file(const char *config_wwids_file)
 		stack;
 
 	log_debug("multipath wwids read %d from %s", count, config_wwids_file);
+	*entries = count;
 }
 
 int dev_mpath_init(const char *config_wwids_file)
@@ -92,6 +258,10 @@ int dev_mpath_init(const char *config_wwids_file)
 	struct dm_pool *mem;
 	struct dm_hash_table *minor_tab;
 	struct dm_hash_table *wwid_tab;
+	int entries = 0;
+
+	dm_list_init(&_ignored);
+	dm_list_init(&_ignored_exceptions);
 
 	if (!(mem = dm_pool_create("mpath", 256))) {
 		log_error("mpath pool creation failed.");
@@ -104,7 +274,7 @@ int dev_mpath_init(const char *config_wwids_file)
 		return 0;
 	}
 
-	_hash_mem = mem;
+	_wwid_mem = mem;
 	_minor_hash_tab = minor_tab;
 
 	/* multipath_wwids_file="" disables the use of the file */
@@ -116,16 +286,24 @@ int dev_mpath_init(const char *config_wwids_file)
 	if (!(wwid_tab = dm_hash_create(110))) {
 		log_error("mpath hash table creation failed.");
 		dm_hash_destroy(_minor_hash_tab);
-		dm_pool_destroy(_hash_mem);
+		dm_pool_destroy(_wwid_mem);
 		_minor_hash_tab = NULL;
-		_hash_mem = NULL;
+		_wwid_mem = NULL;
 		return 0;
 	}
 
 	_wwid_hash_tab = wwid_tab;
 
-	if (config_wwids_file)
-		_read_wwid_file(config_wwids_file);
+	if (config_wwids_file) {
+		_read_wwid_file(config_wwids_file, &entries);
+		_read_wwid_exclusions();
+	}
+
+	if (!entries) {
+		/* reading dev wwids is skipped with null wwid_hash_tab */
+		dm_hash_destroy(_wwid_hash_tab);
+		_wwid_hash_tab = NULL;
+	}
 
 	return 1;
 }
@@ -136,12 +314,12 @@ void dev_mpath_exit(void)
 		dm_hash_destroy(_minor_hash_tab);
 	if (_wwid_hash_tab)
 		dm_hash_destroy(_wwid_hash_tab);
-	if (_hash_mem)
-		dm_pool_destroy(_hash_mem);
+	if (_wwid_mem)
+		dm_pool_destroy(_wwid_mem);
 
 	_minor_hash_tab = NULL;
 	_wwid_hash_tab = NULL;
-	_hash_mem = NULL;
+	_wwid_mem = NULL;
 }
 
 
@@ -272,10 +450,12 @@ static int _dev_is_mpath_component_udev(struct device *dev)
 }
 #endif
 
-static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev)
+/* mpath_devno is major:minor of the dm multipath device currently using the component dev. */
+
+static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev,
+					 int primary_result, dev_t primary_dev, dev_t *mpath_devno)
 {
 	struct dev_types *dt = cmd->dev_types;
-	const char *part_name;
 	const char *name;               /* e.g. "sda" for "/dev/sda" */
 	char link_path[PATH_MAX];       /* some obscure, unpredictable sysfs path */
 	char holders_path[PATH_MAX];    /* e.g. "/sys/block/sda/holders/" */
@@ -289,25 +469,15 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	int dm_dev_major;
 	int dm_dev_minor;
 	struct stat info;
-	dev_t primary_dev;
 	int is_mpath_component = 0;
 
-	/* multipathing is only known to exist for SCSI or NVME devices */
-	if (!major_is_scsi_device(dt, dev_major) && !dev_is_nvme(dt, dev))
-		return 0;
-
-	switch (dev_get_primary_dev(dt, dev, &primary_dev)) {
+	switch (primary_result) {
 
 	case 2: /* The dev is partition. */
-		part_name = dev_name(dev); /* name of original dev for log_debug msg */
 
 		/* gets "foo" for "/dev/foo" where "/dev/foo" comes from major:minor */
 		if (!(name = _get_sysfs_name_by_devt(sysfs_dir, primary_dev, link_path, sizeof(link_path))))
 			return_0;
-
-		log_debug_devs("%s: Device is a partition, using primary "
-			       "device %s for mpath component detection",
-			       part_name, name);
 		break;
 
 	case 1: /* The dev is already a primary dev. Just continue with the dev. */
@@ -426,50 +596,104 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	if (closedir(dr))
 		stack;
 
+	if (is_mpath_component)
+		*mpath_devno = MKDEV(dm_dev_major, dm_dev_minor);
 	return is_mpath_component;
 }
 
-static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev)
+static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev,
+			     int primary_result, dev_t primary_dev)
 {
-	char sysbuf[PATH_MAX] = { 0 };
-	char *wwid;
-	long look;
+	char idbuf[DEV_WWID_SIZE] = { 0 };
+	struct dev_wwid *dw;
+	char *wwid, *full_wwid;
 
 	if (!_wwid_hash_tab)
 		return 0;
 
-	if (!read_sys_block(cmd, dev, "device/wwid", sysbuf, sizeof(sysbuf)))
-		return 0;
+	/*
+	 * Check the primary device, not the partition.
+	 */
+	if (primary_result == 2) {
+		if (!(dev = dev_cache_get_by_devt(cmd, primary_dev))) {
+			log_debug("dev_is_mpath_component %s no primary dev", dev_name(dev));
+			return 0;
+		}
+	}
 
-	if (!sysbuf[0])
+	/*
+	 * sysfs wwid uses format: naa.<value>, eui.<value>, t10.<value>.
+	 * multipath wwids file uses format: 3<value>, 2<value>, 1<value>.
+	 *
+	 * We omit the type prefix before looking up.  The multipath/wwids
+	 * values in the wwid_hash_tab have the initial character removed.
+	 *
+	 * There's no type prefix for "scsi name string" type 8 ids.
+	 *
+	 * First try looking up any wwids that have already been read.
+	 */
+lookup:
+	dm_list_iterate_items(dw, &dev->wwids) {
+		if (dw->type == 1 || dw->type == 2 || dw->type == 3)
+			wwid = &dw->id[4];
+		else
+			wwid = dw->id;
+
+		if (dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid))) {
+			full_wwid = dw->id;
+			goto found;
+		}
+	}
+
+	/*
+	 * The id from sysfs wwid may not be the id used by multipath,
+	 * or a device may not have a vpd_pg83 file (e.g. nvme).
+	 */
+
+	if (!(dev->flags & DEV_ADDED_VPD_WWIDS) && dev_read_vpd_wwids(cmd, dev))
+		goto lookup;
+
+	if (!(dev->flags & DEV_ADDED_SYS_WWID) && dev_read_sys_wwid(cmd, dev, idbuf, sizeof(idbuf), &dw)) {
+		if (dw->type == 1 || dw->type == 2 || dw->type == 3)
+			wwid = &dw->id[4];
+		else
+			wwid = dw->id;
+
+		if (dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid))) {
+			full_wwid = dw->id;
+			goto found;
+		}
+	}
+
+	return 0;
+
+ found:
+	log_debug_devs("dev_is_mpath_component %s %s in wwids file", dev_name(dev), full_wwid);
+	return 1;
+}
+
+int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev, dev_t *holder_devno)
+{
+	struct dev_types *dt = cmd->dev_types;
+	int primary_result;
+	dev_t primary_dev;
+
+	/*
+	 * multipath only uses SCSI or NVME devices
+	 */
+	if (!major_is_scsi_device(dt, MAJOR(dev->dev)) && !dev_is_nvme(dt, dev))
 		return 0;
 
 	/*
-	 * sysfs prints wwid as <typestr>.<value>
-	 * multipath wwid uses '3'<value>
-	 * does "<typestr>." always correspond to "3"?
+	 * primary_result 2: dev is a partition, primary_dev is the whole device
+	 * primary_result 1: dev is a whole device
 	 */
-	if (!(wwid = strchr(sysbuf, '.')))
-		return 0;
+	primary_result = dev_get_primary_dev(dt, dev, &primary_dev);
 
-	/* skip the type and dot, just as '3' was skipped from wwids entry */
-	wwid++;
-	
-	look = (long) dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid));
-
-	if (look) {
-		log_debug_devs("dev_is_mpath_component %s multipath wwid %s", dev_name(dev), wwid);
-		return 1;
-	}
-	return 0;
-}
-
-int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
-{
-	if (_dev_is_mpath_component_sysfs(cmd, dev) == 1)
+	if (_dev_is_mpath_component_sysfs(cmd, dev, primary_result, primary_dev, holder_devno) == 1)
 		goto found;
 
-	if (_dev_in_wwid_file(cmd, dev))
+	if (_dev_in_wwid_file(cmd, dev, primary_result, primary_dev))
 		goto found;
 
 	if (external_device_info_source() == DEV_EXT_UDEV) {
@@ -477,8 +701,85 @@ int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
 			goto found;
 	}
 
+	/*
+	 * TODO: save the result of this function in dev->flags and use those
+	 * flags on repeated calls to avoid repeating the work multiple times
+	 * for the same device when there are partitions on the device.
+	 */
+
 	return 0;
 found:
 	return 1;
 }
 
+const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev)
+{
+	char slaves_path[PATH_MAX];
+	char wwid_path[PATH_MAX];
+	char sysbuf[PATH_MAX] = { 0 };
+	char *slave_name;
+	const char *wwid = NULL;
+	struct stat info;
+	DIR *dr;
+	struct dirent *de;
+
+	/* /sys/dev/block/253:7/slaves/sda/device/wwid */
+
+	if (dm_snprintf(slaves_path, sizeof(slaves_path), "%sdev/block/%d:%d/slaves",
+			dm_sysfs_dir(), (int)MAJOR(dev->dev), (int)MINOR(dev->dev)) < 0) {
+		log_warn("Sysfs path to check mpath components is too long.");
+		return NULL;
+	}
+
+	if (stat(slaves_path, &info))
+		return NULL;
+
+	if (!S_ISDIR(info.st_mode)) {
+		log_warn("Path %s is not a directory.", slaves_path);
+		return NULL;
+	}
+
+	/* Get wwid from first component */
+
+	if (!(dr = opendir(slaves_path))) {
+		log_debug("Device %s has no slaves dir", dev_name(dev));
+		return NULL;
+	}
+
+	while ((de = readdir(dr))) {
+		if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, ".."))
+			continue;
+
+		/* slave_name "sda" */
+		slave_name = de->d_name;
+
+		/* read /sys/block/sda/device/wwid */
+
+		if (dm_snprintf(wwid_path, sizeof(wwid_path), "%sblock/%s/device/wwid",
+       				dm_sysfs_dir(), slave_name) < 0) {
+			log_warn("Failed to create sysfs wwid path for %s", slave_name);
+			continue;
+		}
+
+		get_sysfs_value(wwid_path, sysbuf, sizeof(sysbuf), 0);
+		if (!sysbuf[0])
+			continue;
+
+		if (strstr(sysbuf, "scsi_debug")) {
+			int i;
+			for (i = 0; i < strlen(sysbuf); i++) {
+				if (sysbuf[i] == ' ')
+					sysbuf[i] = '_';
+			}
+		}
+
+		if ((wwid = dm_pool_strdup(cmd->mem, sysbuf)))
+			break;
+	}
+	if (closedir(dr))
+		stack;
+
+	return wwid;
+}
+
+

lib/device/dev-swap.c

@@ -42,9 +42,6 @@ int dev_is_swap(struct cmd_context *cmd, struct device *dev, uint64_t *offset_fo
 	unsigned page;
 	int ret = 0;
 
-	if (!scan_bcache)
-		return -EAGAIN;
-
 	if (!dev_get_size(dev, &size)) {
 		stack;
 		return -1;

lib/device/dev-type.c

@@ -25,7 +25,7 @@
 #include "device_mapper/misc/dm-ioctl.h"
 
 #ifdef BLKID_WIPING_SUPPORT
-#include <blkid.h>
+#include <blkid/blkid.h>
 #endif
 
 #ifdef UDEV_SYNC_SUPPORT
@@ -548,7 +548,7 @@ static int _has_sys_partition(struct device *dev)
 	int minor = (int) MINOR(dev->dev);
 
 	/* check if dev is a partition */
-	if (dm_snprintf(path, sizeof(path), "%s/dev/block/%d:%d/partition",
+	if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/partition",
 			dm_sysfs_dir(), major, minor) < 0) {
 		log_warn("WARNING: %s: partition path is too long.", dev_name(dev));
 		return 0;
@@ -674,11 +674,6 @@ static int _dev_is_partitioned_native(struct dev_types *dt, struct device *dev)
 {
 	int r;
 
-	if (!scan_bcache) {
-		log_error(INTERNAL_ERROR "dev_is_partitioned_native requires i/o.");
-		return -1;
-	}
-
 	/* Unpartitioned DASD devices are not supported. */
 	if ((MAJOR(dev->dev) == dt->dasd_major) && dasd_is_cdl_formatted(dev))
 		return 1;
@@ -780,7 +775,7 @@ int dev_get_primary_dev(struct dev_types *dt, struct device *dev, dev_t *result)
 	 * - basename ../../block/md0/md0  = md0
 	 * Parent's 'dev' sysfs attribute  = /sys/block/md0/dev
 	 */
-	if (dm_snprintf(path, sizeof(path), "%s/dev/block/%d:%d",
+	if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d",
 			dm_sysfs_dir(), major, minor) < 0) {
 		log_warn("WARNING: %s: major:minor sysfs path is too long.", dev_name(dev));
 		return 0;
@@ -792,7 +787,7 @@ int dev_get_primary_dev(struct dev_types *dt, struct device *dev, dev_t *result)
 
 	temp_path[size] = '\0';
 
-	if (dm_snprintf(path, sizeof(path), "%s/block/%s/dev",
+	if (dm_snprintf(path, sizeof(path), "%sblock/%s/dev",
 			dm_sysfs_dir(), basename(dirname(temp_path))) < 0) {
 		log_warn("WARNING: sysfs path for %s is too long.",
 			 basename(dirname(temp_path)));
@@ -828,26 +823,143 @@ out:
 }
 
 #ifdef BLKID_WIPING_SUPPORT
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size)
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs)
 {
-	char *block_size_str = NULL;
+	blkid_probe probe = NULL;
+	const char *type_str = NULL, *size_str = NULL;
+	size_t len;
+	int ret = 1;
+	int rc;
 
-	if ((block_size_str = blkid_get_tag_value(NULL, "BLOCK_SIZE", pathname))) {
-		*fs_block_size = (uint32_t)atoi(block_size_str);
-		free(block_size_str);
-		log_debug("Found blkid BLOCK_SIZE %u for fs on %s", *fs_block_size, pathname);
-		return 1;
-	} else {
-		log_debug("No blkid BLOCK_SIZE for fs on %s", pathname);
-		*fs_block_size = 0;
+	if (!(probe = blkid_new_probe_from_filename(pathname))) {
+		log_error("Failed libblkid probe setup for %s", pathname);
 		return 0;
 	}
+
+	blkid_probe_enable_superblocks(probe, 1);
+	blkid_probe_set_superblocks_flags(probe,
+			BLKID_SUBLKS_LABEL | BLKID_SUBLKS_LABELRAW |
+			BLKID_SUBLKS_UUID | BLKID_SUBLKS_UUIDRAW |
+			BLKID_SUBLKS_TYPE | BLKID_SUBLKS_SECTYPE |
+			BLKID_SUBLKS_USAGE | BLKID_SUBLKS_VERSION |
+#ifdef BLKID_SUBLKS_FSINFO
+			BLKID_SUBLKS_FSINFO |
+#endif
+			BLKID_SUBLKS_MAGIC);
+	rc = blkid_do_safeprobe(probe);
+	if (rc < 0) {
+		log_debug("Failed libblkid probe for %s", pathname);
+		ret = 0;
+		goto out;
+	} else if (rc == 1) {
+		/* no file system on the device */
+		log_debug("No file system found on %s.", pathname);
+		if (nofs)
+			*nofs = 1;
+		goto out;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "TYPE", &type_str, &len) && len && type_str) {
+		if (fstype)
+			strncpy(fstype, type_str, FSTYPE_MAX);
+	} else {
+		/* any difference from blkid_do_safeprobe rc=1? */
+		log_debug("No file system type on %s.", pathname);
+		if (nofs)
+			*nofs = 1;
+		goto out;
+	}
+
+	if (fs_block_size_bytes) {
+		if (!blkid_probe_lookup_value(probe, "BLOCK_SIZE", &size_str, &len) && len && size_str)
+			*fs_block_size_bytes = atoi(size_str);
+		else
+			*fs_block_size_bytes = 0;
+	}
+
+	log_debug("Found blkid fstype %s fsblocksize %s on %s",
+		  type_str ?: "none", size_str ?: "unused", pathname);
+out:
+	blkid_free_probe(probe);
+	return ret;
 }
+
+int fs_get_blkid(const char *pathname, struct fs_info *fsi)
+{
+	blkid_probe probe = NULL;
+	const char *str;
+	size_t len;
+	uint64_t fslastblock = 0;
+	unsigned int fsblocksize = 0;
+	int rc;
+
+	if (!(probe = blkid_new_probe_from_filename(pathname))) {
+		log_error("Failed libblkid probe setup for %s", pathname);
+		return 0;
+	}
+
+	blkid_probe_enable_superblocks(probe, 1);
+	blkid_probe_set_superblocks_flags(probe,
+			BLKID_SUBLKS_LABEL | BLKID_SUBLKS_LABELRAW |
+			BLKID_SUBLKS_UUID | BLKID_SUBLKS_UUIDRAW |
+			BLKID_SUBLKS_TYPE | BLKID_SUBLKS_SECTYPE |
+			BLKID_SUBLKS_USAGE | BLKID_SUBLKS_VERSION |
+#ifdef BLKID_SUBLKS_FSINFO
+			BLKID_SUBLKS_FSINFO |
+#endif
+			BLKID_SUBLKS_MAGIC);
+	rc = blkid_do_safeprobe(probe);
+	if (rc < 0) {
+		log_error("Failed libblkid probe for %s", pathname);
+		blkid_free_probe(probe);
+		return 0;
+	} else if (rc == 1) {
+		/* no file system on the device */
+		log_print("No file system found on %s.", pathname);
+		fsi->nofs = 1;
+		blkid_free_probe(probe);
+		return 1;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "TYPE", &str, &len) && len)
+		strncpy(fsi->fstype, str, sizeof(fsi->fstype)-1);
+	else {
+		/* any difference from blkid_do_safeprobe rc=1? */
+		log_print("No file system type on %s.", pathname);
+		fsi->nofs = 1;
+		blkid_free_probe(probe);
+		return 1;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "BLOCK_SIZE", &str, &len) && len)
+		fsi->fs_block_size_bytes = atoi(str);
+
+	if (!blkid_probe_lookup_value(probe, "FSLASTBLOCK", &str, &len) && len)
+		fslastblock = strtoull(str, NULL, 0);
+
+	if (!blkid_probe_lookup_value(probe, "FSBLOCKSIZE", &str, &len) && len)
+		fsblocksize = (unsigned int)atoi(str);
+
+	blkid_free_probe(probe);
+
+	if (fslastblock && fsblocksize)
+		fsi->fs_last_byte = fslastblock * fsblocksize;
+
+	log_debug("libblkid TYPE %s BLOCK_SIZE %d FSLASTBLOCK %llu FSBLOCKSIZE %u fs_last_byte %llu",
+		  fsi->fstype, fsi->fs_block_size_bytes, (unsigned long long)fslastblock, fsblocksize,
+		  (unsigned long long)fsi->fs_last_byte);
+	return 1;
+}
+
 #else
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size)
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs)
 {
 	log_debug("Disabled blkid BLOCK_SIZE for fs.");
-	*fs_block_size = 0;
+	return 0;
+}
+int fs_get_blkid(const char *pathname, struct fs_info *fsi)
+{
+	log_debug("Disabled blkid for fs info.");
 	return 0;
 }
 #endif
@@ -966,6 +1078,9 @@ static int _wipe_known_signatures_with_blkid(struct device *dev, const char *nam
 
 	/* TODO: Should we check for valid dev - _dev_is_valid(dev)? */
 
+	if (dm_list_empty(&dev->aliases))
+		goto_out;
+
 	if (!(probe = blkid_new_probe_from_filename(dev_name(dev)))) {
 		log_error("Failed to create a new blkid probe for device %s.", dev_name(dev));
 		goto out;
@@ -1097,7 +1212,7 @@ int wipe_known_signatures(struct cmd_context *cmd, struct device *dev,
 static int _snprintf_attr(char *buf, size_t buf_size, const char *sysfs_dir,
 			 const char *attribute, dev_t dev)
 {
-	if (dm_snprintf(buf, buf_size, "%s/dev/block/%d:%d/%s", sysfs_dir,
+	if (dm_snprintf(buf, buf_size, "%sdev/block/%d:%d/%s", sysfs_dir,
 			(int)MAJOR(dev), (int)MINOR(dev),
 			attribute) < 0) {
 		log_warn("WARNING: sysfs path for %s attribute is too long.", attribute);

lib/device/dev-type.h

@@ -18,6 +18,7 @@
 #include "lib/device/device.h"
 #include "lib/display/display.h"
 #include "lib/label/label.h"
+#include "lib/device/filesystem.h"
 
 #define NUMBER_OF_MAJORS 4096
 
@@ -58,11 +59,13 @@ int major_is_scsi_device(struct dev_types *dt, int major);
 
 /* Signature/superblock recognition with position returned where found. */
 int dev_is_md_component(struct cmd_context *cmd, struct device *dev, uint64_t *sb, int full);
-int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev);
+int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev, dev_t *mpath_devno);
 int dev_is_swap(struct cmd_context *cmd, struct device *dev, uint64_t *signature, int full);
 int dev_is_luks(struct cmd_context *cmd, struct device *dev, uint64_t *signature, int full);
 int dasd_is_cdl_formatted(struct device *dev);
 
+const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev);
+
 int dev_is_lvm1(struct device *dev, char *buf, int buflen);
 int dev_is_pool(struct device *dev, char *buf, int buflen);
 
@@ -99,7 +102,9 @@ int dev_is_nvme(struct dev_types *dt, struct device *dev);
 
 int dev_is_lv(struct device *dev);
 
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size);
+#define FSTYPE_MAX 16
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs);
+int fs_get_blkid(const char *pathname, struct fs_info *fsi);
 
 int dev_is_used_by_active_lv(struct cmd_context *cmd, struct device *dev, int *used_by_lv_count,
 			     char **used_by_dm_name, char **used_by_vg_uuid, char **used_by_lv_uuid);

lib/device/dev_util.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2013 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/device/device.h"
+
+int device_id_list_remove(struct dm_list *list, struct device *dev)
+{
+	struct device_id_list *dil;
+
+	dm_list_iterate_items(dil, list) {
+		if (dil->dev == dev) {
+			dm_list_del(&dil->list);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+struct device_id_list *device_id_list_find_dev(struct dm_list *list, struct device *dev)
+{
+	struct device_id_list *dil;
+
+	dm_list_iterate_items(dil, list) {
+		if (dil->dev == dev)
+			return dil;
+	}
+	return NULL;
+}
+
+int device_list_remove(struct dm_list *list, struct device *dev)
+{
+	struct device_list *devl;
+
+	dm_list_iterate_items(devl, list) {
+		if (devl->dev == dev) {
+			dm_list_del(&devl->list);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+struct device_list *device_list_find_dev(struct dm_list *list, struct device *dev)
+{
+	struct device_list *devl;
+
+	dm_list_iterate_items(devl, list) {
+		if (devl->dev == dev)
+			return devl;
+	}
+	return NULL;
+}
+

lib/device/device-types.h

@@ -43,6 +43,7 @@ static const dev_known_type_t _dev_known_types[] = {
 	{"ubd", 16, "User-mode virtual block device"},
 	{"ataraid", 16, "ATA Raid"},
 	{"drbd", 16, "Distributed Replicated Block Device (DRBD)"},
+	{"rbd", 16, "Ceph rados object as a Linux block device"},
 	{"emcpower", 16, "EMC Powerpath"},
 	{"power2", 16, "EMC Powerpath"},
 	{"i2o_block", 16, "i2o Block Disk"},

lib/device/device.h

@@ -32,14 +32,15 @@
 #define DEV_NOT_O_NOATIME	0x00000400	/* Don't use O_NOATIME */
 #define DEV_IN_BCACHE		0x00000800      /* dev fd is open and used in bcache */
 #define DEV_BCACHE_EXCL		0x00001000      /* bcache_fd should be open EXCL */
-#define DEV_FILTER_AFTER_SCAN	0x00002000	/* apply filter after bcache has data */
-#define DEV_FILTER_OUT_SCAN	0x00004000	/* filtered out during label scan */
+#define DEV_ADDED_SYS_WWID	0x00002000      /* wwid has been added from sysfs wwid file */
+#define DEV_ADDED_VPD_WWIDS	0x00004000	/* wwids have been added from vpd_pg83 */
 #define DEV_BCACHE_WRITE	0x00008000      /* bcache_fd is open with RDWR */
 #define DEV_SCAN_FOUND_LABEL	0x00010000      /* label scan read dev and found label */
 #define DEV_IS_MD_COMPONENT	0x00020000	/* device is an md component */
 #define DEV_IS_NVME		0x00040000	/* set if dev is nvme */
 #define DEV_MATCHED_USE_ID	0x00080000	/* matched an entry from cmd->use_devices */
 #define DEV_SCAN_FOUND_NOLABEL	0x00100000	/* label_scan read, passed filters, but no lvm label */
+#define DEV_SCAN_NOT_READ	0x00200000	/* label_scan not able to read dev */
 
 /*
  * Support for external device info.
@@ -58,14 +59,33 @@ struct dev_ext {
 	void *handle;
 };
 
-#define DEV_ID_TYPE_SYS_WWID   0x0001
-#define DEV_ID_TYPE_SYS_SERIAL 0x0002
-#define DEV_ID_TYPE_MPATH_UUID 0x0003
-#define DEV_ID_TYPE_MD_UUID    0x0004
-#define DEV_ID_TYPE_LOOP_FILE  0x0005
-#define DEV_ID_TYPE_CRYPT_UUID 0x0006
-#define DEV_ID_TYPE_LVMLV_UUID 0x0007
-#define DEV_ID_TYPE_DEVNAME    0x0008
+#define DEV_ID_TYPE_SYS_WWID   1
+#define DEV_ID_TYPE_SYS_SERIAL 2
+#define DEV_ID_TYPE_MPATH_UUID 3
+#define DEV_ID_TYPE_MD_UUID    4
+#define DEV_ID_TYPE_LOOP_FILE  5
+#define DEV_ID_TYPE_CRYPT_UUID 6
+#define DEV_ID_TYPE_LVMLV_UUID 7
+#define DEV_ID_TYPE_DEVNAME    8
+#define DEV_ID_TYPE_WWID_NAA   9
+#define DEV_ID_TYPE_WWID_EUI  10
+#define DEV_ID_TYPE_WWID_T10  11
+
+/* Max length of WWID_NAA, WWID_EUI, WWID_T10 */
+#define DEV_WWID_SIZE 128
+
+/*
+ * A wwid read from:
+ * /sys/dev/block/%d:%d/device/wwid
+ * /sys/dev/block/%d:%d/wwid
+ * /sys/dev/block/%d:%d/device/vpd_pg83
+ */
+
+struct dev_wwid {
+	struct dm_list list;     /* dev->wwids */
+	int type;                /* 1,2,3 for NAA,EUI,T10 */
+	char id[DEV_WWID_SIZE];  /* includes prefix naa.,eui.,t10. */
+};
 
 /*
  * A device ID of a certain type for a device.
@@ -74,7 +94,7 @@ struct dev_ext {
  */
 
 struct dev_id {
-	struct dm_list list;
+	struct dm_list list;    /* dev->ids */
 	struct device *dev;
 	uint16_t idtype;	/* DEV_ID_TYPE_ */
 	char *idname;		/* id string determined by idtype */
@@ -98,12 +118,18 @@ struct dev_use {
 	char *pvid;
 };
 
+struct dev_use_list {
+	struct dm_list list;
+	struct dev_use *du;
+};
+
 /*
  * All devices in LVM will be represented by one of these.
  * pointer comparisons are valid.
  */
 struct device {
 	struct dm_list aliases;	/* struct dm_str_list */
+	struct dm_list wwids; /* struct dev_wwid, used for multipath component detection */
 	struct dm_list ids; /* struct dev_id, different entries for different idtypes */
 	struct dev_id *id; /* points to the the ids entry being used for this dev */
 	dev_t dev;
@@ -153,6 +179,12 @@ struct device_list {
 	struct device *dev;
 };
 
+struct device_id_list {
+	struct dm_list list;
+	struct device *dev;
+	char pvid[ID_LEN + 1];
+};
+
 struct device_area {
 	struct device *dev;
 	uint64_t start;		/* Bytes */
@@ -203,10 +235,16 @@ struct device *dev_create_file(const char *filename, struct device *dev,
 			       struct dm_str_list *alias, int use_malloc);
 void dev_destroy_file(struct device *dev);
 
-/* Return a valid device name from the alias list; NULL otherwise */
-const char *dev_name_confirmed(struct device *dev, int quiet);
-
 int dev_mpath_init(const char *config_wwids_file);
 void dev_mpath_exit(void);
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids);
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes);
+int parse_vpd_serial(const unsigned char *in, char *out, int outsize);
+
+/* dev_util */
+int device_id_list_remove(struct dm_list *devices, struct device *dev);
+struct device_id_list *device_id_list_find_dev(struct dm_list *devices, struct device *dev);
+int device_list_remove(struct dm_list *devices, struct device *dev);
+struct device_list *device_list_find_dev(struct dm_list *devices, struct device *dev);
 
 #endif

lib/device/device_id.h

@@ -28,18 +28,23 @@ int device_ids_use_devname(struct cmd_context *cmd);
 int device_ids_read(struct cmd_context *cmd);
 int device_ids_write(struct cmd_context *cmd);
 int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid,
-                  const char *idtype_arg, const char *id_arg);
+                  const char *idtype_arg, const char *id_arg, int use_idtype_only);
 void device_id_pvremove(struct cmd_context *cmd, struct device *dev);
 void device_ids_match(struct cmd_context *cmd);
 int device_ids_match_dev(struct cmd_context *cmd, struct device *dev);
+void device_ids_match_device_list(struct cmd_context *cmd);
 void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs, int *device_ids_invalid, int noupdate);
 int device_ids_version_unchanged(struct cmd_context *cmd);
+void device_ids_check_serial(struct cmd_context *cmd, struct dm_list *scan_devs, int *update_needed, int noupdate);
 void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_list, int *search_count, int noupdate);
 const char *device_id_system_read(struct cmd_context *cmd, struct device *dev, uint16_t idtype);
 void device_id_update_vg_uuid(struct cmd_context *cmd, struct volume_group *vg, struct id *old_vg_id);
 
+struct dev_use *get_du_for_devno(struct cmd_context *cmd, dev_t devno);
 struct dev_use *get_du_for_dev(struct cmd_context *cmd, struct device *dev);
 struct dev_use *get_du_for_pvid(struct cmd_context *cmd, const char *pvid);
+struct dev_use *get_du_for_devname(struct cmd_context *cmd, const char *devname);
+struct dev_use *get_du_for_device_id(struct cmd_context *cmd, uint16_t idtype, const char *idname);
 
 char *devices_file_version(void);
 int devices_file_exists(struct cmd_context *cmd);
@@ -54,5 +59,17 @@ void devices_file_exit(struct cmd_context *cmd);
 void unlink_searched_devnames(struct cmd_context *cmd);
 
 int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suffix, char *sysbuf, int sysbufsize);
+int read_sys_block_binary(struct cmd_context *cmd, struct device *dev,
+			  const char *suffix, char *sysbuf, int sysbufsize, int *retlen);
+
+int dev_has_mpath_uuid(struct cmd_context *cmd, struct device *dev, const char **idname_out);
+
+int wwid_type_to_idtype(int wwid_type);
+int idtype_to_wwid_type(int idtype);
+void free_wwids(struct dm_list *ids);
+struct dev_wwid *dev_add_wwid(char *id, int id_type, struct dm_list *ids);
+int dev_read_vpd_wwids(struct cmd_context *cmd, struct device *dev);
+int dev_read_sys_wwid(struct cmd_context *cmd, struct device *dev,
+		      char *buf, int bufsize, struct dev_wwid **dw_out);
 
 #endif

lib/device/filesystem.c

@@ -0,0 +1,548 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/device/device.h"
+#include "lib/device/dev-type.h"
+#include "lib/misc/lvm-exec.h"
+#include "lib/activate/dev_manager.h"
+
+#include <dirent.h>
+#include <mntent.h>
+#include <sys/ioctl.h>
+
+static const char *_lvresize_fs_helper_path;
+
+static const char *_get_lvresize_fs_helper_path(void)
+{
+	if (!_lvresize_fs_helper_path)
+		_lvresize_fs_helper_path = getenv("LVRESIZE_FS_HELPER_PATH");
+
+	if (!_lvresize_fs_helper_path)
+		_lvresize_fs_helper_path = LVRESIZE_FS_HELPER_PATH; /* from configure, usually in /usr/libexec */
+
+	return _lvresize_fs_helper_path;
+}
+
+/*
+ * Set the path of the dm-crypt device, i.e. /dev/dm-N, that is using the LV.
+ */
+static int _get_crypt_path(dev_t lv_devt, char *lv_path, char *crypt_path)
+{
+	char holders_path[PATH_MAX];
+	char *holder_name;
+	DIR *dr;
+	struct stat st;
+	struct dirent *de;
+	int ret = 0;
+
+	if (dm_snprintf(holders_path, sizeof(holders_path), "%sdev/block/%d:%d/holders",
+			dm_sysfs_dir(), (int)MAJOR(lv_devt), (int)MINOR(lv_devt)) < 0)
+		return_0;
+
+	/* If the crypt dev is not active, there will be no LV holder. */
+	if (stat(holders_path, &st)) {
+		log_error("Missing %s for %s", crypt_path, lv_path);
+		return 0;
+	}
+
+	if (!(dr = opendir(holders_path))) {
+		log_error("Cannot open %s", holders_path);
+		return 0;
+	}
+
+	while ((de = readdir(dr))) {
+		if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, ".."))
+			continue;
+
+		holder_name = de->d_name;
+
+		if (strncmp(holder_name, "dm", 2)) {
+			log_error("Unrecognized holder %s of %s", holder_name, lv_path);
+			ret = 0;
+			break;
+		}
+
+		/* We could read the holder's dm uuid to verify it's a crypt dev. */
+
+		if (dm_snprintf(crypt_path, PATH_MAX, "/dev/%s", holder_name) < 0) {
+			ret = 0;
+			stack;
+			break;
+		}
+		ret = 1;
+		break;
+	}
+	closedir(dr);
+	if (ret)
+		log_debug("Found holder %s of %s.", crypt_path, lv_path);
+	else
+		log_debug("No holder in %s", holders_path);
+	return ret;
+}
+
+int lv_crypt_is_active(struct cmd_context *cmd, char *lv_path)
+{
+	char crypt_path[PATH_MAX];
+	struct stat st_lv;
+
+	if (stat(lv_path, &st_lv) < 0) {
+		log_error("Failed to get LV path %s", lv_path);
+		return 0;
+	}
+
+	return _get_crypt_path(st_lv.st_rdev, lv_path, crypt_path);
+}
+
+int fs_get_info(struct cmd_context *cmd, struct logical_volume *lv,
+		struct fs_info *fsi, int include_mount)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	struct stat st_lv;
+	struct stat st_crypt;
+	struct stat st_top;
+	struct stat stme;
+	struct fs_info info;
+	FILE *fme = NULL;
+	struct mntent *me;
+	int fd;
+	int ret;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir,
+			lv->vg->name, lv->name) < 0) {
+		log_error("Couldn't create LV path for %s.", display_lvname(lv));
+		return 0;
+	}
+
+	if (stat(lv_path, &st_lv) < 0) {
+		log_error("Failed to get LV path %s", lv_path);
+		return 0;
+	}
+
+	memset(&info, 0, sizeof(info));
+
+	if (!fs_get_blkid(lv_path, &info)) {
+		log_error("No file system info from blkid for %s", display_lvname(lv));
+		return 0;
+	}
+
+	if (fsi->nofs)
+		return 1;
+
+	/*
+	 * If there's a LUKS dm-crypt layer over the LV, then
+	 * return fs info from that layer, setting needs_crypt
+	 * to indicate a crypt layer between the fs and LV.
+	 */
+	if (!strcmp(info.fstype, "crypto_LUKS")) {
+		if (!_get_crypt_path(st_lv.st_rdev, lv_path, crypt_path)) {
+			log_error("Cannot find active LUKS dm-crypt device using %s.",
+				  display_lvname(lv));
+			return 0;
+		}
+
+		if (stat(crypt_path, &st_crypt) < 0) {
+			log_error("Failed to get crypt path %s", crypt_path);
+			return 0;
+		}
+
+		memset(&info, 0, sizeof(info));
+
+		log_print("Checking crypt device %s on LV %s.",
+			  crypt_path, display_lvname(lv));
+
+		if ((fd = open(crypt_path, O_RDONLY)) < 0) {
+			log_error("Failed to open crypt path %s", crypt_path);
+			return 0;
+		}
+		if (ioctl(fd, BLKGETSIZE64, &info.crypt_dev_size_bytes) < 0) {
+			log_error("Failed to get crypt device size %s", crypt_path);
+			close(fd);
+			return 0;
+		}
+		close(fd);
+
+		if (!fs_get_blkid(crypt_path, &info)) {
+			log_error("No file system info from blkid for dm-crypt device %s on LV %s.",
+				  crypt_path, display_lvname(lv));
+			return 0;
+		}
+		*fsi = info;
+		fsi->needs_crypt = 1;
+		fsi->crypt_devt = st_crypt.st_rdev;
+		memcpy(fsi->fs_dev_path, crypt_path, PATH_MAX);
+		st_top = st_crypt;
+
+		if (!get_crypt_table_offset(st_crypt.st_rdev, &fsi->crypt_offset_bytes)) {
+			log_error("Failed to get crypt data offset.");
+			return 0;
+		}
+	} else {
+		*fsi = info;
+		memcpy(fsi->fs_dev_path, lv_path, PATH_MAX);
+		st_top = st_lv;
+	}
+
+	if (!include_mount)
+		return 1;
+
+	if (!(fme = setmntent("/etc/mtab", "r")))
+		return_0;
+
+	ret = 1;
+
+	while ((me = getmntent(fme))) {
+		if (strcmp(me->mnt_type, fsi->fstype))
+			continue;
+		if (me->mnt_dir[0] != '/')
+			continue;
+		if (me->mnt_fsname[0] != '/')
+			continue;
+		if (stat(me->mnt_dir, &stme) < 0)
+			continue;
+		if (stme.st_dev != st_top.st_rdev)
+			continue;
+
+		log_debug("fs_get_info %s is mounted \"%s\"", fsi->fs_dev_path, me->mnt_dir);
+		fsi->mounted = 1;
+		strncpy(fsi->mount_dir, me->mnt_dir, PATH_MAX-1);
+	}
+	endmntent(fme);
+
+	fsi->unmounted = !fsi->mounted;
+	return ret;
+}
+
+int fs_mount_state_is_misnamed(struct cmd_context *cmd, struct logical_volume *lv, char *lv_path, char *fstype)
+{
+	FILE *fp;
+	char proc_line[PATH_MAX];
+	char proc_fstype[FSTYPE_MAX];
+	char proc_devpath[PATH_MAX];
+	char proc_mntpath[PATH_MAX];
+	char lv_mapper_path[PATH_MAX];
+	char mntent_mount_dir[PATH_MAX];
+	char *dm_name;
+	struct stat st_lv;
+	struct stat stme;
+	FILE *fme = NULL;
+	struct mntent *me;
+	int renamed = 0;
+	int found_dir = 0;
+	int found_dev = 0;
+	int dev_match, dir_match;
+
+	if (stat(lv_path, &st_lv) < 0) {
+		log_error("Failed to get LV path %s", lv_path);
+		return 0;
+	}
+
+	/*
+	 * If LVs have been renamed while their file systems were mounted, then
+	 * inconsistencies appear in the device path and mount point info
+	 * provided by getmntent and /proc/mounts.  If there's any
+	 * inconsistency or duplication of info for the LV name or the mount
+	 * point, then give up and don't try fs resize which is likely to fail
+	 * due to kernel problems where mounts reference old device names
+	 * causing fs resizing tools to fail.
+	 */
+
+	if (!(fme = setmntent("/etc/mtab", "r")))
+		return_0;
+
+	while ((me = getmntent(fme))) {
+		if (strcmp(me->mnt_type, fstype))
+			continue;
+		if (me->mnt_dir[0] != '/')
+			continue;
+		if (me->mnt_fsname[0] != '/')
+			continue;
+		if (stat(me->mnt_dir, &stme) < 0)
+			continue;
+		if (stme.st_dev != st_lv.st_rdev)
+			continue;
+		dm_strncpy(mntent_mount_dir, me->mnt_dir, sizeof(mntent_mount_dir));
+	}
+	endmntent(fme);
+
+	if (!(dm_name = dm_build_dm_name(cmd->mem, lv->vg->name, lv->name, NULL)))
+		return_0;
+
+	if ((dm_snprintf(lv_mapper_path, sizeof(lv_mapper_path), "%s/%s", dm_dir(), dm_name) < 0))
+		return_0;
+
+	if (!(fp = fopen("/proc/mounts", "r")))
+		return_0;
+
+	while (fgets(proc_line, sizeof(proc_line), fp)) {
+		if (proc_line[0] != '/')
+			continue;
+		if (sscanf(proc_line, "%s %s %s", proc_devpath, proc_mntpath, proc_fstype) != 3)
+			continue;
+		if (strcmp(fstype, proc_fstype))
+			continue;
+
+		dir_match = !strcmp(mntent_mount_dir, proc_mntpath);
+		dev_match = !strcmp(lv_mapper_path, proc_devpath);
+
+		if (dir_match)
+			found_dir++;
+		if (dev_match)
+			found_dev++;
+
+		if (dir_match != dev_match) {
+			log_error("LV %s mounted at %s may have been renamed (from %s).",
+				  lv_mapper_path, proc_mntpath, proc_devpath);
+			renamed = 1;
+		}
+	}
+
+	if (fclose(fp))
+		stack;
+
+	/*
+	 * Don't try resizing if:
+	 * - different device names apppear for the mount point
+	 *   (LVs probably renamed while mounted), or
+	 * - the mount point for the LV appears multiple times, or
+	 * - the LV device is listed for multiple mounts. 
+	 */
+	if (renamed) {
+		log_error("File system resizing not supported: fs utilities do not support renamed devices.");
+		return 1;
+	}
+	/* These two are likely detected as renamed, but include checks in case. */
+	if (found_dir > 1) {
+		log_error("File system resizing not supported: %s appears more than once in /proc/mounts.", mntent_mount_dir);
+		return 1;
+	}
+	if (found_dev > 1) {
+		log_error("File system resizing not supported: %s appears more than once in /proc/mounts.", lv_mapper_path);
+		return 1;
+	}
+	return 0;
+}
+
+#define FS_CMD_MAX_ARGS 16
+
+int crypt_resize_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+			uint64_t newsize_bytes_fs)
+{
+	char crypt_path[PATH_MAX];
+	char newsize_str[16] = { 0 };
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(newsize_str, sizeof(newsize_str), "%llu", (unsigned long long)newsize_bytes_fs) < 0)
+		return_0;
+
+	if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--cryptresize";
+	argv[++args] = "--cryptpath";
+	argv[++args] = crypt_path;
+	argv[++args] = "--newsizebytes";
+	argv[++args] = newsize_str;
+	argv[++args] = NULL;
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to resize crypt dev with lvresize_fs_helper.");
+		return 0;
+	}
+
+	return 1;
+}
+
+/*
+ * The helper script does the following steps for reduce:
+ * devpath = $cryptpath ? $cryptpath : $lvpath
+ * if needs_unmount
+ * 	umount $mountdir 
+ * if needs_fsck
+ * 	e2fsck -f -p $devpath
+ * if needs_mount
+ * 	mount $devpath $tmpdir
+ * if $fstype == "ext"
+ * 	resize2fs $devpath $newsize_kb
+ * if needs_crypt
+ * 	cryptsetup resize --size $newsize_sectors $cryptpath
+ *
+ * Note: when a crypt layer is included, newsize_bytes_fs is smaller
+ * than newsize_bytes_lv because of the crypt header.
+ */
+
+int fs_reduce_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		     uint64_t newsize_bytes_fs, char *fsmode)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	char newsize_str[16] = { 0 };
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	char *devpath;
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(newsize_str, sizeof(newsize_str), "%llu", (unsigned long long)newsize_bytes_fs) < 0)
+		return_0;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir, lv->vg->name, lv->name) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--fsreduce";
+	argv[++args] = "--fstype";
+	argv[++args] = fsi->fstype;
+	argv[++args] = "--lvpath";
+	argv[++args] = lv_path;
+
+	if (newsize_bytes_fs) {
+		argv[++args] = "--newsizebytes";
+		argv[++args] = newsize_str;
+	}
+	if (fsi->mounted) {
+		argv[++args] = "--mountdir";
+		argv[++args] = fsi->mount_dir;
+	}
+
+	if (fsi->needs_unmount)
+		argv[++args] = "--unmount";
+	if (fsi->needs_mount)
+		argv[++args] = "--mount";
+	if (fsi->needs_fsck)
+		argv[++args] = "--fsck";
+
+	if (fsi->needs_crypt) {
+		if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+			return_0;
+		argv[++args] = "--cryptresize";
+		argv[++args] = "--cryptpath";
+		argv[++args] = crypt_path;
+	}
+
+	/*
+	 * fsmode manage means the fs should be remounted after
+	 * resizing if it was unmounted.
+	 */
+	if (fsi->needs_unmount && !strcmp(fsmode, "manage"))
+		argv[++args] = "--remount";
+
+	argv[++args] = NULL;
+
+	devpath = fsi->needs_crypt ? crypt_path : (char *)display_lvname(lv);
+
+	log_print("Reducing file system %s to %s (%llu bytes) on %s...",
+		  fsi->fstype, display_size(cmd, newsize_bytes_fs/512),
+		  (unsigned long long)newsize_bytes_fs, devpath);
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to reduce file system with lvresize_fs_helper.");
+		return 0;
+	}
+
+	log_print("Reduced file system %s on %s.", fsi->fstype, devpath);
+
+	return 1;
+}
+
+/*
+ * The helper script does the following steps for extend:
+ * devpath = $cryptpath ? $cryptpath : $lvpath
+ * if needs_unmount
+ * 	umount $mountdir 
+ * if needs_fsck
+ * 	e2fsck -f -p $devpath
+ * if needs_crypt
+ * 	cryptsetup resize $cryptpath
+ * if needs_mount
+ * 	mount $devpath $tmpdir
+ * if $fstype == "ext"
+ * 	resize2fs $devpath
+ * if $fstype == "xfs"
+ * 	xfs_growfs $devpath
+ *
+ * Note: when a crypt layer is included, newsize_bytes_fs is smaller
+ * than newsize_bytes_lv because of the crypt header.
+ */
+
+int fs_extend_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		     uint64_t newsize_bytes_fs, char *fsmode)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	char *devpath;
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir, lv->vg->name, lv->name) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--fsextend";
+	argv[++args] = "--fstype";
+	argv[++args] = fsi->fstype;
+	argv[++args] = "--lvpath";
+	argv[++args] = lv_path;
+
+	if (fsi->mounted) {
+		argv[++args] = "--mountdir";
+		argv[++args] = fsi->mount_dir;
+	}
+
+	if (fsi->needs_unmount)
+		argv[++args] = "--unmount";
+	if (fsi->needs_mount)
+		argv[++args] = "--mount";
+	if (fsi->needs_fsck)
+		argv[++args] = "--fsck";
+
+	if (fsi->needs_crypt) {
+		if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+			return_0;
+		argv[++args] = "--cryptresize";
+		argv[++args] = "--cryptpath";
+		argv[++args] = crypt_path;
+	}
+
+	/*
+	 * fsmode manage means the fs should be remounted after
+	 * resizing if it was unmounted.
+	 */
+	if (fsi->needs_unmount && !strcmp(fsmode, "manage"))
+		argv[++args] = "--remount";
+
+	argv[++args] = NULL;
+
+	devpath = fsi->needs_crypt ? crypt_path : (char *)display_lvname(lv);
+
+	log_print("Extending file system %s to %s (%llu bytes) on %s...",
+		  fsi->fstype, display_size(cmd, newsize_bytes_fs/512),
+		  (unsigned long long)newsize_bytes_fs, devpath);
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to extend file system with lvresize_fs_helper.");
+		return 0;
+	}
+
+	log_print("Extended file system %s on %s.", fsi->fstype, devpath);
+
+	return 1;
+}
+

lib/device/filesystem.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _FILESYSTEM_H
+#define _FILESYSTEM_H
+
+#define FSTYPE_MAX 16
+
+struct fs_info {
+	char fstype[FSTYPE_MAX];
+	char mount_dir[PATH_MAX];
+	char fs_dev_path[PATH_MAX]; /* usually lv dev, can be crypt dev */
+	unsigned int fs_block_size_bytes; /* 512 or 4k */
+	uint64_t fs_last_byte; /* last byte on the device used by the fs */
+	uint32_t crypt_offset_bytes; /* offset in bytes of crypt data on LV */
+	dev_t crypt_devt; /* dm-crypt device between the LV and FS */
+	uint64_t crypt_dev_size_bytes;
+
+	unsigned nofs:1;
+	unsigned unmounted:1;
+	unsigned mounted:1;
+	/* for resizing */
+	unsigned needs_reduce:1;
+	unsigned needs_extend:1;
+	unsigned needs_fsck:1;
+	unsigned needs_unmount:1;
+	unsigned needs_mount:1;
+	unsigned needs_crypt:1;
+};
+
+int fs_get_info(struct cmd_context *cmd, struct logical_volume *lv,
+                struct fs_info *fsi, int include_mount);
+
+int fs_extend_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes, char *fsmode);
+int fs_reduce_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes, char *fsmode);
+int crypt_resize_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes_fs);
+
+int fs_mount_state_is_misnamed(struct cmd_context *cmd, struct logical_volume *lv, char *lv_path, char *fstype);
+int lv_crypt_is_active(struct cmd_context *cmd, char *lv_path);
+
+#endif

lib/device/online.c

@@ -0,0 +1,561 @@
+/*
+ * Copyright (C) 2021 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/device/device.h"
+#include "lib/device/online.h"
+
+#include <dirent.h>
+
+/*
+ * file contains:
+ * <major>:<minor>\n
+ * vg:<vgname>\n
+ * dev:<devname>\n\0
+ *
+ * It's possible that vg and dev may not exist.
+ */
+
+static int _copy_pvid_file_field(const char *field, char *buf, int bufsize, char *out, int outsize)
+{
+	char *p;
+	int i = 0;
+	
+	if (!(p = strstr(buf, field)))
+		return 0;
+
+	p += strlen(field);
+
+	while (1) {
+		if (*p == '\n')
+			break;
+		if (*p == '\0')
+			break;
+
+		if (p >= (buf + bufsize))
+			return 0;
+		if (i >= outsize-1)
+			return 0;
+
+		out[i] = *p;
+
+		i++;
+		p++;
+	}
+
+	return i ? 1 : 0;
+}
+
+#define MAX_PVID_FILE_SIZE 512
+
+int online_pvid_file_read(char *path, int *major, int *minor, char *vgname, char *devname)
+{
+	char buf[MAX_PVID_FILE_SIZE] = { 0 };
+	int fd, rv;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0) {
+		log_warn("Failed to open %s", path);
+		return 0;
+	}
+
+	rv = read(fd, buf, sizeof(buf) - 1);
+	if (close(fd))
+		log_sys_debug("close", path);
+	if (!rv || rv < 0) {
+		log_warn("No info in %s", path);
+		return 0;
+	}
+	buf[rv] = 0; /* \0 terminated buffer */
+
+	if (sscanf(buf, "%d:%d", major, minor) != 2) {
+		log_warn("No device numbers in %s", path);
+		return 0;
+	}
+
+	if (vgname) {
+		if (!strstr(buf, "vg:")) {
+			log_debug("No vgname in %s", path);
+			vgname[0] = '\0';
+			goto copy_dev;
+		}
+
+		if (!_copy_pvid_file_field("vg:", buf, MAX_PVID_FILE_SIZE, vgname, NAME_LEN)) {
+			log_warn("Ignoring invalid vg field in %s", path);
+			vgname[0] = '\0';
+			goto copy_dev;
+		}
+
+		if (!validate_name(vgname)) {
+			log_warn("Ignoring invalid vgname in %s (%s)", path, vgname);
+			vgname[0] = '\0';
+			goto copy_dev;
+		}
+	}
+
+ copy_dev:
+	if (devname) {
+		if (!strstr(buf, "dev:")) {
+			log_debug("No devname in %s", path);
+			devname[0] = '\0';
+			goto out;
+		}
+
+		if (!_copy_pvid_file_field("dev:", buf, MAX_PVID_FILE_SIZE, devname, NAME_LEN)) {
+			log_warn("Ignoring invalid devname field in %s", path);
+			devname[0] = '\0';
+			goto out;
+		}
+
+		if (strncmp(devname, "/dev/", 5)) {
+			log_warn("Ignoring invalid devname in %s (%s)", path, devname);
+			devname[0] = '\0';
+			goto out;
+		}
+	}
+ out:
+	return 1;
+}
+
+void free_po_list(struct dm_list *list)
+{
+	struct pv_online *po, *po2;
+
+	dm_list_iterate_items_safe(po, po2, list) {
+		dm_list_del(&po->list);
+		free(po);
+	}
+}
+
+int get_pvs_online(struct dm_list *pvs_online, const char *vgname)
+{
+	char path[PATH_MAX];
+	char file_vgname[NAME_LEN];
+	char file_devname[NAME_LEN];
+	DIR *dir;
+	struct dirent *de;
+	struct pv_online *po;
+	int file_major = 0, file_minor = 0;
+
+	if (!(dir = opendir(PVS_ONLINE_DIR)))
+		return 0;
+
+	while ((de = readdir(dir))) {
+		if (de->d_name[0] == '.')
+			continue;
+
+		if (strlen(de->d_name) != ID_LEN)
+			continue;
+
+		memset(path, 0, sizeof(path));
+		snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, de->d_name);
+
+		file_major = 0;
+		file_minor = 0;
+		memset(file_vgname, 0, sizeof(file_vgname));
+		memset(file_devname, 0, sizeof(file_devname));
+
+		if (!online_pvid_file_read(path, &file_major, &file_minor, file_vgname, file_devname))
+			continue;
+
+		if (vgname && strcmp(file_vgname, vgname))
+			continue;
+
+		if (!(po = zalloc(sizeof(*po))))
+			continue;
+
+		memcpy(po->pvid, de->d_name, ID_LEN);
+		if (file_major || file_minor)
+			po->devno = MKDEV(file_major, file_minor);
+		if (file_vgname[0])
+			strncpy(po->vgname, file_vgname, NAME_LEN);
+		if (file_devname[0])
+			strncpy(po->devname, file_devname, NAME_LEN);
+
+		log_debug("Found PV online %s for VG %s %s", path, vgname, file_devname);
+		dm_list_add(pvs_online, &po->list);
+	}
+
+	if (closedir(dir))
+		log_sys_debug("closedir", PVS_ONLINE_DIR);
+
+	log_debug("Found PVs online %d for %s", dm_list_size(pvs_online), vgname ?: "all");
+
+	return 1;
+}
+
+/*
+ * When a PV goes offline, remove the vg online file for that VG
+ * (even if other PVs for the VG are still online).  This means
+ * that the vg will be activated again when it becomes complete.
+ */
+
+void online_vg_file_remove(const char *vgname)
+{
+	char path[PATH_MAX];
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", VGS_ONLINE_DIR, vgname) < 0) {
+		log_error("Path %s/%s is too long.", VGS_ONLINE_DIR, vgname);
+		return;
+	}
+
+	log_debug("Unlink vg online: %s", path);
+
+	if (unlink(path) && (errno != ENOENT))
+		log_sys_debug("unlink", path);
+}
+
+int online_vg_file_create(struct cmd_context *cmd, const char *vgname)
+{
+	char path[PATH_MAX];
+	int fd;
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", VGS_ONLINE_DIR, vgname) < 0) {
+		log_error_pvscan(cmd, "Path %s/%s is too long.", VGS_ONLINE_DIR, vgname);
+		return 0;
+	}
+
+	log_debug("Create vg online: %s", path);
+
+	fd = open(path, O_CREAT | O_EXCL | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
+	if (fd < 0) {
+		log_debug("Failed to create %s: %d", path, errno);
+		return 0;
+	}
+
+	/* We don't care about syncing, these files are not even persistent. */
+
+	if (close(fd))
+		log_sys_debug("close", path);
+
+	return 1;
+}
+
+int online_pvid_file_create(struct cmd_context *cmd, struct device *dev, const char *vgname)
+{
+	char path[PATH_MAX];
+	char buf[MAX_PVID_FILE_SIZE] = { 0 };
+	char file_vgname[NAME_LEN];
+	char file_devname[NAME_LEN];
+	char devname[NAME_LEN];
+	int devnamelen;
+	int file_major = 0, file_minor = 0;
+	int major, minor;
+	int fd;
+	int rv;
+	int len;
+	int len1 = 0;
+	int len2 = 0;
+	int len3 = 0;
+
+	major = (int)MAJOR(dev->dev);
+	minor = (int)MINOR(dev->dev);
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, dev->pvid) < 0) {
+		log_error_pvscan(cmd, "Path %s/%s is too long.", PVS_ONLINE_DIR, dev->pvid);
+		return 0;
+	}
+
+	if ((len1 = dm_snprintf(buf, sizeof(buf), "%d:%d\n", major, minor)) < 0) {
+		log_error_pvscan(cmd, "Cannot create online file path for %s %d:%d.", dev_name(dev), major, minor);
+		return 0;
+	}
+
+	if (vgname) {
+		if ((len2 = dm_snprintf(buf + len1, sizeof(buf) - len1, "vg:%s\n", vgname)) < 0) {
+			log_print("Incomplete online file for %s %d:%d vg %s.", dev_name(dev), major, minor, vgname);
+			/* can still continue without vgname */
+			len2 = 0;
+		}
+	}
+
+	devnamelen = dm_snprintf(devname, sizeof(devname), "%s", dev_name(dev));
+	if ((devnamelen > 5) && (devnamelen < NAME_LEN-1)) {
+		if ((len3 = dm_snprintf(buf + len1 + len2, sizeof(buf) - len1 - len2, "dev:%s\n", devname)) < 0) {
+			log_print("Incomplete devname in online file for %s.", dev_name(dev));
+			/* can continue without devname */
+			len3 = 0;
+		}
+	}
+
+	len = len1 + len2 + len3;
+
+	log_debug("Create pv online: %s %d:%d %s", path, major, minor, dev_name(dev));
+
+	fd = open(path, O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR);
+	if (fd < 0) {
+		if (errno == EEXIST)
+			goto check_duplicate;
+		log_error_pvscan(cmd, "Failed to create online file for %s path %s error %d", dev_name(dev), path, errno);
+		return 0;
+	}
+
+	while (len > 0) {
+		rv = write(fd, buf, len);
+		if (rv < 0) {
+			/* file exists so it still works in part */
+			log_warn("Cannot write online file for %s to %s error %d",
+				  dev_name(dev), path, errno);
+			if (close(fd))
+				log_sys_debug("close", path);
+			return 1;
+		}
+		len -= rv;
+	}
+
+	/* We don't care about syncing, these files are not even persistent. */
+
+	if (close(fd))
+		log_sys_debug("close", path);
+
+	return 1;
+
+check_duplicate:
+
+	/*
+	 * If a PVID online file already exists for this PVID, check if the
+	 * file contains a different device number, and if so we may have a
+	 * duplicate PV.
+	 *
+	 * FIXME: disable autoactivation of the VG somehow?
+	 * The VG may or may not already be activated when a dupicate appears.
+	 * Perhaps write a new field in the pv online or vg online file?
+	 */
+
+	memset(file_vgname, 0, sizeof(file_vgname));
+	memset(file_devname, 0, sizeof(file_devname));
+
+	online_pvid_file_read(path, &file_major, &file_minor, file_vgname, file_devname);
+
+	if ((file_major == major) && (file_minor == minor)) {
+		log_debug("Existing online file for %d:%d", major, minor);
+		return 1;
+	}
+
+	/* Don't know how vgname might not match, but it's not good so fail. */
+
+	if ((file_major != major) || (file_minor != minor))
+		log_error_pvscan(cmd, "PV %s %d:%d is duplicate for PVID %s on %d:%d %s.",
+			         dev_name(dev), major, minor, dev->pvid, file_major, file_minor, file_devname);
+
+	if (file_vgname[0] && vgname && strcmp(file_vgname, vgname))
+		log_error_pvscan(cmd, "PV %s has unexpected VG %s vs %s.",
+			         dev_name(dev), vgname, file_vgname);
+
+	return 0;
+}
+
+int online_pvid_file_exists(const char *pvid)
+{
+	char path[PATH_MAX] = { 0 };
+	struct stat buf;
+	int rv;
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, pvid) < 0) {
+		log_debug(INTERNAL_ERROR "Path %s/%s is too long.", PVS_ONLINE_DIR, pvid);
+		return 0;
+	}
+
+	log_debug("Check pv online: %s", path);
+
+	rv = stat(path, &buf);
+	if (!rv) {
+		log_debug("Check pv online %s: yes", pvid);
+		return 1;
+	}
+	log_debug("Check pv online %s: no", pvid);
+	return 0;
+}
+
+int get_pvs_lookup(struct dm_list *pvs_online, const char *vgname)
+{
+	char lookup_path[PATH_MAX] = { 0 };
+	char path[PATH_MAX] = { 0 };
+	char line[64];
+	char pvid[ID_LEN + 1] __attribute__((aligned(8))) = { 0 };
+	char file_vgname[NAME_LEN];
+	char file_devname[NAME_LEN];
+	struct pv_online *po;
+	int file_major = 0, file_minor = 0;
+	FILE *fp;
+
+	if (dm_snprintf(lookup_path, sizeof(path), "%s/%s", PVS_LOOKUP_DIR, vgname) < 0)
+		return_0;
+
+	if (!(fp = fopen(lookup_path, "r")))
+		return_0;
+
+	while (fgets(line, sizeof(line), fp)) {
+		memcpy(pvid, line, ID_LEN);
+		if (strlen(pvid) != ID_LEN)
+			goto_bad;
+
+		memset(path, 0, sizeof(path));
+		snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, pvid);
+
+		file_major = 0;
+		file_minor = 0;
+		memset(file_vgname, 0, sizeof(file_vgname));
+		memset(file_devname, 0, sizeof(file_devname));
+
+		if (!online_pvid_file_read(path, &file_major, &file_minor, file_vgname, file_devname))
+			goto_bad;
+
+		/*
+		 * PVs without metadata will not have a vgname in their pvid
+		 * file, but the purpose of using the lookup file is that we
+		 * know the PV is for this VG even without the pvid vgname
+		 * field.
+		 */
+		if (vgname && file_vgname[0] && strcmp(file_vgname, vgname)) {
+			/* Should never happen */
+			log_error("Incorrect VG lookup file %s PVID %s %s.", vgname, pvid, file_vgname);
+			goto_bad;
+		}
+
+		if (!(po = zalloc(sizeof(*po))))
+			goto_bad;
+
+		memcpy(po->pvid, pvid, ID_LEN);
+		if (file_major || file_minor)
+			po->devno = MKDEV(file_major, file_minor);
+		if (file_vgname[0])
+			strncpy(po->vgname, file_vgname, NAME_LEN-1);
+		if (file_devname[0])
+			strncpy(po->devname, file_devname, NAME_LEN-1);
+
+		log_debug("Found PV online lookup %s for VG %s on %s", path, vgname, file_devname);
+		dm_list_add(pvs_online, &po->list);
+	}
+
+	log_debug("Found PVs online lookup %d for %s", dm_list_size(pvs_online), vgname);
+
+	fclose(fp);
+	return 1;
+
+bad:
+	free_po_list(pvs_online);
+	fclose(fp);
+	return 0;
+}
+
+void online_dir_setup(struct cmd_context *cmd)
+{
+	struct stat st;
+	int rv;
+
+	if (!stat(DEFAULT_RUN_DIR, &st))
+		goto do_pvs;
+
+	log_debug("Creating run_dir.");
+	dm_prepare_selinux_context(DEFAULT_RUN_DIR, S_IFDIR);
+	rv = mkdir(DEFAULT_RUN_DIR, 0755);
+	dm_prepare_selinux_context(NULL, 0);
+
+	if ((rv < 0) && stat(DEFAULT_RUN_DIR, &st))
+		log_error_pvscan(cmd, "Failed to create %s %d", DEFAULT_RUN_DIR, errno);
+
+do_pvs:
+	if (!stat(PVS_ONLINE_DIR, &st))
+		goto do_vgs;
+
+	log_debug("Creating pvs_online_dir.");
+	dm_prepare_selinux_context(PVS_ONLINE_DIR, S_IFDIR);
+	rv = mkdir(PVS_ONLINE_DIR, 0755);
+	dm_prepare_selinux_context(NULL, 0);
+
+	if ((rv < 0) && stat(PVS_ONLINE_DIR, &st))
+		log_error_pvscan(cmd, "Failed to create %s %d", PVS_ONLINE_DIR, errno);
+
+do_vgs:
+	if (!stat(VGS_ONLINE_DIR, &st))
+		goto do_lookup;
+
+	log_debug("Creating vgs_online_dir.");
+	dm_prepare_selinux_context(VGS_ONLINE_DIR, S_IFDIR);
+	rv = mkdir(VGS_ONLINE_DIR, 0755);
+	dm_prepare_selinux_context(NULL, 0);
+
+	if ((rv < 0) && stat(VGS_ONLINE_DIR, &st))
+		log_error_pvscan(cmd, "Failed to create %s %d", VGS_ONLINE_DIR, errno);
+
+do_lookup:
+	if (!stat(PVS_LOOKUP_DIR, &st))
+		return;
+
+	log_debug("Creating pvs_lookup_dir.");
+	dm_prepare_selinux_context(PVS_LOOKUP_DIR, S_IFDIR);
+	rv = mkdir(PVS_LOOKUP_DIR, 0755);
+	dm_prepare_selinux_context(NULL, 0);
+
+	if ((rv < 0) && stat(PVS_LOOKUP_DIR, &st))
+		log_error_pvscan(cmd, "Failed to create %s %d", PVS_LOOKUP_DIR, errno);
+}
+
+void online_lookup_file_remove(const char *vgname)
+{
+	char path[PATH_MAX];
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_LOOKUP_DIR, vgname) < 0) {
+		log_error("Path %s/%s is too long.", PVS_LOOKUP_DIR, vgname);
+		return;
+	}
+
+	log_debug("Unlink pvs_lookup: %s", path);
+
+	if (unlink(path) && (errno != ENOENT))
+		log_sys_debug("unlink", path);
+}
+
+static int _online_pvid_file_remove(char *pvid)
+{
+	char path[PATH_MAX];
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, pvid) < 0)
+		return_0;
+	if (!unlink(path))
+		return 1;
+	return 0;
+}
+
+/*
+ * Reboot automatically clearing tmpfs on /run is the main method of removing
+ * online files.  It's important to note that removing the online files for a
+ * VG is not a technical requirement for anything and could easily be skipped
+ * if it had any downside.  It's only done to clean up the space used in /run
+ * by the online files, e.g. if there happens to be an extreme amount of
+ * vgcreate/pvscan/vgremove between reboots that are leaving a large number of
+ * useless online files consuming tmpfs space.
+ */
+void online_vgremove(struct volume_group *vg)
+{
+	char pvid[ID_LEN + 1] __attribute__((aligned(8))) = { 0 };
+	struct pv_list *pvl;
+
+	/*
+	 * online files may not exist for the vg if there has been no
+	 * pvscans or autoactivation.
+	 */
+
+	online_vg_file_remove(vg->name);
+	online_lookup_file_remove(vg->name);
+
+	dm_list_iterate_items(pvl, &vg->pvs) {
+		memcpy(pvid, &pvl->pv->id.uuid, ID_LEN);
+		_online_pvid_file_remove(pvid);
+	}
+}
+

lib/device/online.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2021 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _ONLINE_H
+#define _ONLINE_H
+
+struct pv_online {
+	struct dm_list list;
+	struct device *dev;
+	dev_t devno;
+	char pvid[ID_LEN + 1];
+	char vgname[NAME_LEN];
+	char devname[NAME_LEN];
+};
+
+/*
+ * Avoid a duplicate pvscan[%d] prefix when logging to the journal.
+ * FIXME: this should probably replace if (udevoutput) with
+ * if (log_journal & LOG_JOURNAL_OUTPUT)
+ */
+#define log_print_pvscan(cmd, fmt, args...) \
+do \
+	if (cmd->udevoutput) \
+		log_print(fmt, ##args); \
+	else \
+		log_print("pvscan[%d] " fmt, getpid(), ##args); \
+while (0)
+
+#define log_error_pvscan(cmd, fmt, args...) \
+do \
+	if (cmd->udevoutput) \
+		log_error(fmt, ##args); \
+	else \
+		log_error("pvscan[%d] " fmt, getpid(), ##args); \
+while (0)
+
+int online_pvid_file_read(char *path, int *major, int *minor, char *vgname, char *devname);
+int online_vg_file_create(struct cmd_context *cmd, const char *vgname);
+void online_vg_file_remove(const char *vgname);
+int online_pvid_file_create(struct cmd_context *cmd, struct device *dev, const char *vgname);
+int online_pvid_file_exists(const char *pvid);
+void online_dir_setup(struct cmd_context *cmd);
+int get_pvs_online(struct dm_list *pvs_online, const char *vgname);
+int get_pvs_lookup(struct dm_list *pvs_online, const char *vgname);
+void free_po_list(struct dm_list *list);
+void online_lookup_file_remove(const char *vgname);
+void online_vgremove(struct volume_group *vg);
+
+#endif

lib/device/parse_vpd.c

@@ -0,0 +1,258 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/device/device.h"
+#include "lib/device/device_id.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <limits.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <assert.h>
+
+/*
+ * Replace series of spaces with a single _.
+ */
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int in_space = 0;
+	int retlen = 0;
+	int j = 0;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		if (!in[i])
+			break;
+		if (j >= (out_bytes - 2))
+			break;
+		/* skip leading spaces */
+		if (!retlen && (in[i] == ' '))
+			continue;
+		/* skip non-ascii non-printable characters */
+		if (!isascii(in[i]) || !isprint(in[i]))
+			continue;
+		/* skip quote */
+		if (in[i] == '"')
+			continue;
+		/* replace one or more spaces with _ */
+		if (in[i] == ' ') {
+			in_space = 1;
+			continue;
+		}
+		/* spaces are finished so insert _ */
+		if (in_space) {
+			out[j++] = '_';
+			in_space = 0;
+			retlen++;
+		}
+		out[j++] = in[i];
+		retlen++;
+	}
+	return retlen;
+}
+
+static int _to_hex(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int off = 0;
+	int num;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		num = sprintf((char *)out + off, "%02x", in[i]);
+		if (num < 0)
+			break;
+		off += num;
+		if (off + 2 >= out_bytes)
+			break;
+	}
+	return off;
+}
+
+#define ID_BUFSIZE 1024
+
+/*
+ * based on linux kernel function
+ */
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids)
+{
+	char id[ID_BUFSIZE];
+	unsigned char tmp_str[ID_BUFSIZE];
+	const unsigned char *d, *cur_id_str;
+	size_t id_len = ID_BUFSIZE;
+	int id_size = -1;
+	int type;
+	uint8_t cur_id_size = 0;
+
+	memset(id, 0, ID_BUFSIZE);
+	for (d = vpd_data + 4;
+	     d < vpd_data + vpd_datalen;
+	     d += d[3] + 4) {
+		memset(tmp_str, 0, sizeof(tmp_str));
+
+		switch (d[1] & 0xf) {
+		case 0x1:
+			/* T10 Vendor ID */
+			cur_id_size = d[3];
+			if (cur_id_size + 4 > id_len)
+				cur_id_size = id_len - 4;
+			cur_id_str = d + 4;
+			format_t10_id(cur_id_str, cur_id_size, tmp_str, sizeof(tmp_str));
+			id_size = snprintf(id, ID_BUFSIZE, "t10.%s", tmp_str);
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 1, ids);
+			break;
+		case 0x2:
+			/* EUI-64 */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 12:
+				_to_hex(cur_id_str, 12, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 2, ids);
+			break;
+		case 0x3:
+			/* NAA */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 3, ids);
+			break;
+		case 0x8:
+			/* SCSI name string */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			if (cur_id_size >= id_len)
+				cur_id_size = id_len - 1;
+			memcpy(id, cur_id_str, cur_id_size);
+			id_size = cur_id_size;
+
+			/*
+			 * if naa or eui ids are provided as scsi names,
+			 * consider them to be naa/eui types.
+			 */
+			if (!memcmp(id, "eui.", 4))
+				type = 2;
+			else if (!memcmp(id, "naa.", 4))
+				type = 3;
+			else
+				type = 8;
+
+			/*
+			 * Not in the kernel version, copying multipath code,
+			 * which checks if this string begins with naa or eui
+			 * and if so does tolower() on the chars.
+			 */
+			if ((type == 2) || (type == 3)) {
+				int i;
+				for (i = 0; i < strlen(id); i++)
+					id[i] = tolower(id[i]);
+			}
+			dev_add_wwid(id, type, ids);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return id_size;
+}
+
+int parse_vpd_serial(const unsigned char *in, char *out, int outsize)
+{
+	uint8_t len_buf[2] __attribute__((aligned(8))) = { 0 };;
+	size_t len;
+
+	/* parsing code from multipath tools */
+	/* ignore in[0] and in[1] */
+	/* len is in[2] and in[3] */
+	/* serial begins at in[4] */
+
+	len_buf[0] = in[2];
+	len_buf[1] = in[3];
+	len = len_buf[0] << 8 | len_buf[1];
+
+	if (outsize == 0)
+		return 0;
+
+	if (len > DEV_WWID_SIZE)
+		len = DEV_WWID_SIZE;
+	/*
+	 * Strip leading and trailing whitespace
+	 */
+	while (len > 0 && in[len + 3] == ' ')
+		--len;
+	while (len > 0 && in[4] == ' ') {
+		++in;
+		--len;
+	}
+
+	if (len >= outsize)
+                len = outsize - 1;
+
+	if (len > 0) {
+		memcpy(out, in + 4, len);
+		out[len] = '\0';
+	}
+	return len;
+}
+

lib/filters/filter-internal.c

@@ -1,86 +0,0 @@
-/*
- * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
- *
- * This file is part of LVM2.
- *
- * This copyrighted material is made available to anyone wishing to use,
- * modify, copy, or redistribute it subject to the terms and conditions
- * of the GNU Lesser General Public License v.2.1.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include "base/memory/zalloc.h"
-#include "lib/misc/lib.h"
-#include "lib/filters/filter.h"
-
-static DM_LIST_INIT(_allow_devs);
-
-int internal_filter_allow(struct dm_pool *mem, struct device *dev)
-{
-	struct device_list *devl;
-
-	if (!(devl = dm_pool_alloc(mem, sizeof(*devl)))) {
-		log_error("device_list element allocation failed");
-		return 0;
-	}
-	devl->dev = dev;
-
-	dm_list_add(&_allow_devs, &devl->list);
-	return 1;
-}
-
-void internal_filter_clear(void)
-{
-	dm_list_init(&_allow_devs);
-}
-
-static int _passes_internal(struct cmd_context *cmd, struct dev_filter *f __attribute__((unused)),
-			    struct device *dev, const char *use_filter_name)
-{
-	struct device_list *devl;
-
-	dev->filtered_flags &= ~DEV_FILTERED_INTERNAL;
-
-	if (!internal_filtering())
-		return 1;
-	
-	dm_list_iterate_items(devl, &_allow_devs) {
-		if (devl->dev == dev)
-			return 1;
-	}
-	
-	dev->filtered_flags |= DEV_FILTERED_INTERNAL;
-	log_debug_devs("%s: Skipping for internal filtering.", dev_name(dev));
-	return 0;
-}
-
-static void _destroy(struct dev_filter *f)
-{
-	if (f->use_count)
-		log_error(INTERNAL_ERROR "Destroying internal filter while in use %u times.", f->use_count);
-
-	free(f);
-}
-
-struct dev_filter *internal_filter_create(void)
-{
-	struct dev_filter *f;
-
-	if (!(f = zalloc(sizeof(*f)))) {
-		log_error("md filter allocation failed");
-		return NULL;
-	}
-
-	f->passes_filter = _passes_internal;
-	f->destroy = _destroy;
-	f->use_count = 0;
-	f->name = "internal";
-
-	log_debug_devs("Internal filter initialised.");
-
-	return f;
-}
-

lib/filters/filter-md.c

@@ -99,14 +99,6 @@ static int _passes_md_filter(struct cmd_context *cmd, struct dev_filter *f __att
 		return 1;
 
 	ret = dev_is_md_component(cmd, dev, NULL, cmd->use_full_md_check);
-
-	if (ret == -EAGAIN) {
-		/* let pass, call again after scan */
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		log_debug_devs("filter md deferred %s", dev_name(dev));
-		return 1;
-	}
-
 	if (ret == 0)
 		return 1;
 

lib/filters/filter-mpath.c

@@ -15,18 +15,41 @@
 #include "base/memory/zalloc.h"
 #include "lib/misc/lib.h"
 #include "lib/filters/filter.h"
+#include "lib/device/device_id.h"
 
 #ifdef __linux__
 
 #include <dirent.h>
 
+static int _lvmdevices_update_msg;
+
 static int _ignore_mpath_component(struct cmd_context *cmd, struct dev_filter *f, struct device *dev, const char *use_filter_name)
 {
+	dev_t mpath_devno = 0;
+
 	dev->filtered_flags &= ~DEV_FILTERED_MPATH_COMPONENT;
 
-	if (dev_is_mpath_component(cmd, dev)) {
+	if (dev_is_mpath_component(cmd, dev, &mpath_devno)) {
 		log_debug_devs("%s: Skipping mpath component device", dev_name(dev));
 		dev->filtered_flags |= DEV_FILTERED_MPATH_COMPONENT;
+
+		/*
+		 * Warn about misconfig where an mpath component is
+		 * in the devices file, but its mpath device is not.
+		 */
+		if ((dev->flags & DEV_MATCHED_USE_ID) && mpath_devno) {
+			if (!get_du_for_devno(cmd, mpath_devno)) {
+				struct device *mpath_dev = dev_cache_get_by_devt(cmd, mpath_devno);
+				log_warn("WARNING: devices file is missing %s (%d:%d) using multipath component %s.",
+					 mpath_dev ? dev_name(mpath_dev) : "unknown",
+					 (int)MAJOR(mpath_devno), (int)MINOR(mpath_devno), dev_name(dev));
+				if (!_lvmdevices_update_msg && strcmp(get_cmd_name(), "lvmdevices")) {
+					log_warn("See lvmdevices --update for devices file update.");
+					_lvmdevices_update_msg = 1;
+				}
+			}
+		}
+
 		return 0;
 	}
 

lib/filters/filter-partitioned.c

@@ -30,14 +30,6 @@ static int _passes_partitioned_filter(struct cmd_context *cmd, struct dev_filter
 	dev->filtered_flags &= ~DEV_FILTERED_PARTITIONED;
 
 	ret = dev_is_partitioned(cmd, dev);
-
-	if (ret == -EAGAIN) {
-		/* let pass, call again after scan */
-		log_debug_devs("filter partitioned deferred %s", dev_name(dev));
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		return 1;
-	}
-
 	if (ret) {
 		if (dev->ext.src == DEV_EXT_NONE)
 			log_debug_devs(MSG_SKIPPING, dev_name(dev));