pvscan: use alternate device names from DEVLINKS to check filter

The filter may contains a symlink for the device, and that symlink
may not yet be created when our udev rule runs pvscan --cache using
the kernel dev name.  The kernel dev name does not match the symlink
name in the filter, so pvscan will ignore the device.  udev sets
the DEVLINKS env variable to a list of link names for the device
that will be created, so check the filter with that set of names.

.gitignore

@@ -16,6 +16,10 @@
 *.sw*
 *~
 
+# gcov files:
+*.gcda
+*.gcno
+
 .export.sym
 .exported_symbols_generated
 .gdb_history
@@ -39,9 +43,11 @@ make.tmpl
 
 coverity/coverity_model.xml
 
-# gcov files:
-*.gcda
-*.gcno
+/libdm/.symver_check
+
+daemons/clvmd
+daemons/dmfilemapd
+daemons/lvmetad/
 
 tools/man-generator
 tools/man-generator.c
@@ -102,6 +108,8 @@ test/api/thin_percent.t
 test/api/vglist.t
 test/api/vgtest.t
 test/lib/aux
+test/lib/cache-mq.profile
+test/lib/cache-smq.profile
 test/lib/check
 test/lib/clvmd
 test/lib/dm-version-expected
@@ -111,6 +119,7 @@ test/lib/dmstats
 test/lib/fail
 test/lib/flavour-ndev-cluster
 test/lib/flavour-ndev-cluster-lvmpolld
+test/lib/flavour-ndev-devicesfile
 test/lib/flavour-ndev-lvmetad
 test/lib/flavour-ndev-lvmetad-lvmpolld
 test/lib/flavour-ndev-lvmpolld
@@ -120,6 +129,7 @@ test/lib/flavour-udev-cluster-lvmpolld
 test/lib/flavour-udev-lvmetad
 test/lib/flavour-udev-lvmetad-lvmpolld
 test/lib/flavour-udev-lvmlockd-dlm
+test/lib/flavour-udev-lvmlockd-idm
 test/lib/flavour-udev-lvmlockd-sanlock
 test/lib/flavour-udev-lvmlockd-test
 test/lib/flavour-udev-lvmpolld
@@ -132,8 +142,11 @@ test/lib/lvm
 test/lib/lvm-wrapper
 test/lib/lvmchange
 test/lib/lvmdbusd.profile
+test/lib/lvmdevices
 test/lib/lvmetad
 test/lib/lvmpolld
+test/lib/lvm_import_vdo
+test/lib/lvm_vdo_wrapper
 test/lib/not
 test/lib/paths
 test/lib/paths-common
@@ -143,5 +156,7 @@ test/lib/test
 test/lib/thin-performance.profile
 test/lib/utils
 test/lib/version-expected
+test/lib/vgimportdevices
+
 test/unit/dmraid_t.c
 test/unit/unit-test

Makefile.in

@@ -47,7 +47,7 @@ include $(top_srcdir)/base/Makefile
 include $(top_srcdir)/device_mapper/Makefile
 include $(top_srcdir)/test/unit/Makefile
 
-lib: libdaemon $(BASE_TARGET) $(DEVICE_MAPPER_TARGET)
+lib: include libdaemon $(BASE_TARGET) $(DEVICE_MAPPER_TARGET)
 daemons: lib libdaemon tools
 scripts: lib
 tools: lib libdaemon

TESTING

@@ -24,7 +24,7 @@ You MUST disable (or mask) any LVM daemons:
 For running cluster tests, we are using singlenode locking. Pass
 `--with-clvmd=singlenode` to configure.
 
-NOTE: This is useful only for testing, and should not be used in produciton
+NOTE: This is useful only for testing, and should not be used in production
 code.
 
 To run D-Bus daemon tests, existing D-Bus session is required.

VERSION

@@ -1 +1 @@
-2.03.16(2)-git (2022-02-07)
+2.03.19(2)-git (2022-12-22)

VERSION_DM

@@ -1 +1 @@
-1.02.185-git (2022-02-07)
+1.02.191-git (2022-12-22)

WHATS_NEW

@@ -1,5 +1,42 @@
-Version 2.03.16 - 
+version 2.03.19 - 
 ====================================
+  Fix and improve runtime memory size detection for VDO volumes.
+
+version 2.03.18 - 22nd december 2022
+====================================
+  Fix issues reported by coverity scan.
+  Fix warning for thin pool overprovisioning on lvextend (2.03.17).
+  Add support for writecache metadata_only and pause_writeback settings.
+  Fix missing error messages in lvmdbusd.
+
+Version 2.03.17 - 10th November 2022
+====================================
+  Add new options (--fs, --fsmode) for FS handling when resizing LVs.
+  Fix 'lvremove -S|--select LV' to not also remove its historical LV right away.
+  Fix lv_active field type to binary so --select and --binary applies properly.
+  Switch to use mallinfo2 and use it only with glibc.
+  Error out in lvm shell if using a cmd argument not supported in the shell.
+  Fix lvm shell's lastlog command to report previous pre-command failures.
+  Extend VDO and VDOPOOL without flushing and locking fs.
+  Add --valuesonly option to lvmconfig to print only values without keys.
+  Updates configure with recent autoconf tooling.
+  Fix lvconvert --test --type vdo-pool execution.
+  Add json_std output format for more JSON standard compliant version of output.
+  Fix vdo_slab_size_mb value for converted VDO volume.
+  Fix many corner cases in device_id, including handling of S/N duplicates.
+  Fix various issues in lvmdbusd.
+
+Version 2.03.16 - 18th May 2022
+===============================
+  Fix segfault when handling selection with historical LVs.
+  Add support --vdosettings with lvcreate, lvconvert, lvchange.
+  Filtering multipath devices respects blacklist setting from multipath
+  configuration.
+  lvmdevices support for removing by device id using --deviceidtype and
+  --deldev.
+  Display writecache block size with lvs -o writecache_block_size.
+  Improve cachesettings description in man lvmcache.
+  Fix lossing of delete message on thin-pool extension.
 
 Version 2.03.15 - 07th February 2022
 ====================================
@@ -160,7 +197,7 @@ Version 2.03.10 - 09th August 2020
 
 Version 2.03.09 - 26th March 2020
 =================================
-  Fix formating of vdopool (vdo_slab_size_mb was smaller by 2 bits).
+  Fix formatting of vdopool (vdo_slab_size_mb was smaller by 2 bits).
   Fix showing of a dm kernel error when uncaching a volume with cachevol.
 
 Version 2.03.08 - 11th February 2020

WHATS_NEW_DM

@@ -1,6 +1,17 @@
-Version 1.02.185 - 
+Version 1.02.191 - 
 =====================================
 
+Version 1.02.189 - 22nd December 2022
+=====================================
+  Improve 'dmsetup create' without given table line with new kernels.
+
+Version 1.02.187 - 10th November 2022
+=====================================
+  Add DM_REPORT_GROUP_JSON_STD for more JSON standard compliant output format.
+
+Version 1.02.185 - 18th May 2022
+================================
+
 Version 1.02.183 - 07th February 2022
 =====================================
   Unmangle UUIDs for DM_DEVICE_LIST ioctl.

aclocal.m4

@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.16.2 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.5 -*- Autoconf -*-
 
-# Copyright (C) 1996-2020 Free Software Foundation, Inc.
+# Copyright (C) 1996-2021 Free Software Foundation, Inc.
 
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -413,7 +413,7 @@ AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
         [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
 ])dnl PKG_HAVE_DEFINE_WITH_MODULES
 
-# Copyright (C) 1999-2020 Free Software Foundation, Inc.
+# Copyright (C) 1999-2021 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -492,7 +492,7 @@ AC_DEFUN([AM_PATH_PYTHON],
   ])
 
   if test "$PYTHON" = :; then
-  dnl Run any user-specified action, or abort.
+    dnl Run any user-specified action, or abort.
     m4_default([$3], [AC_MSG_ERROR([no suitable Python interpreter found])])
   else
 
@@ -501,27 +501,132 @@ AC_DEFUN([AM_PATH_PYTHON],
   dnl trailing zero was eliminated. So now we output just the major
   dnl and minor version numbers, as numbers. Apparently the tertiary
   dnl version is not of interest.
-
+  dnl
   AC_CACHE_CHECK([for $am_display_PYTHON version], [am_cv_python_version],
-    [am_cv_python_version=`$PYTHON -c "import sys; print('%u.%u' % sys.version_info[[:2]])"`])
+    [am_cv_python_version=`$PYTHON -c "import sys; print ('%u.%u' % sys.version_info[[:2]])"`])
   AC_SUBST([PYTHON_VERSION], [$am_cv_python_version])
 
-  dnl Use the values of $prefix and $exec_prefix for the corresponding
-  dnl values of PYTHON_PREFIX and PYTHON_EXEC_PREFIX.  These are made
-  dnl distinct variables so they can be overridden if need be.  However,
-  dnl general consensus is that you shouldn't need this ability.
-
-  AC_SUBST([PYTHON_PREFIX], ['${prefix}'])
-  AC_SUBST([PYTHON_EXEC_PREFIX], ['${exec_prefix}'])
-
-  dnl At times (like when building shared libraries) you may want
+  dnl At times, e.g., when building shared libraries, you may want
   dnl to know which OS platform Python thinks this is.
-
+  dnl
   AC_CACHE_CHECK([for $am_display_PYTHON platform], [am_cv_python_platform],
     [am_cv_python_platform=`$PYTHON -c "import sys; sys.stdout.write(sys.platform)"`])
   AC_SUBST([PYTHON_PLATFORM], [$am_cv_python_platform])
 
-  # Just factor out some code duplication.
+  dnl emacs-page
+  dnl If --with-python-sys-prefix is given, use the values of sys.prefix
+  dnl and sys.exec_prefix for the corresponding values of PYTHON_PREFIX
+  dnl and PYTHON_EXEC_PREFIX. Otherwise, use the GNU ${prefix} and
+  dnl ${exec_prefix} variables.
+  dnl
+  dnl The two are made distinct variables so they can be overridden if
+  dnl need be, although general consensus is that you shouldn't need
+  dnl this separation.
+  dnl
+  dnl Also allow directly setting the prefixes via configure options,
+  dnl overriding any default.
+  dnl
+  if test "x$prefix" = xNONE; then
+    am__usable_prefix=$ac_default_prefix
+  else
+    am__usable_prefix=$prefix
+  fi
+
+  # Allow user to request using sys.* values from Python,
+  # instead of the GNU $prefix values.
+  AC_ARG_WITH([python-sys-prefix],
+  [AS_HELP_STRING([--with-python-sys-prefix],
+                  [use Python's sys.prefix and sys.exec_prefix values])],
+  [am_use_python_sys=:],
+  [am_use_python_sys=false])
+
+  # Allow user to override whatever the default Python prefix is.
+  AC_ARG_WITH([python_prefix],
+  [AS_HELP_STRING([--with-python_prefix],
+                  [override the default PYTHON_PREFIX])],
+  [am_python_prefix_subst=$withval
+   am_cv_python_prefix=$withval
+   AC_MSG_CHECKING([for explicit $am_display_PYTHON prefix])
+   AC_MSG_RESULT([$am_cv_python_prefix])],
+  [
+   if $am_use_python_sys; then
+     # using python sys.prefix value, not GNU
+     AC_CACHE_CHECK([for python default $am_display_PYTHON prefix],
+     [am_cv_python_prefix],
+     [am_cv_python_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.prefix)"`])
+
+     dnl If sys.prefix is a subdir of $prefix, replace the literal value of
+     dnl $prefix with a variable reference so it can be overridden.
+     case $am_cv_python_prefix in
+     $am__usable_prefix*)
+       am__strip_prefix=`echo "$am__usable_prefix" | sed 's|.|.|g'`
+       am_python_prefix_subst=`echo "$am_cv_python_prefix" | sed "s,^$am__strip_prefix,\\${prefix},"`
+       ;;
+     *)
+       am_python_prefix_subst=$am_cv_python_prefix
+       ;;
+     esac
+   else # using GNU prefix value, not python sys.prefix
+     am_python_prefix_subst='${prefix}'
+     am_python_prefix=$am_python_prefix_subst
+     AC_MSG_CHECKING([for GNU default $am_display_PYTHON prefix])
+     AC_MSG_RESULT([$am_python_prefix])
+   fi])
+  # Substituting python_prefix_subst value.
+  AC_SUBST([PYTHON_PREFIX], [$am_python_prefix_subst])
+
+  # emacs-page Now do it all over again for Python exec_prefix, but with yet
+  # another conditional: fall back to regular prefix if that was specified.
+  AC_ARG_WITH([python_exec_prefix],
+  [AS_HELP_STRING([--with-python_exec_prefix],
+                  [override the default PYTHON_EXEC_PREFIX])],
+  [am_python_exec_prefix_subst=$withval
+   am_cv_python_exec_prefix=$withval
+   AC_MSG_CHECKING([for explicit $am_display_PYTHON exec_prefix])
+   AC_MSG_RESULT([$am_cv_python_exec_prefix])],
+  [
+   # no explicit --with-python_exec_prefix, but if
+   # --with-python_prefix was given, use its value for python_exec_prefix too.
+   AS_IF([test -n "$with_python_prefix"],
+   [am_python_exec_prefix_subst=$with_python_prefix
+    am_cv_python_exec_prefix=$with_python_prefix
+    AC_MSG_CHECKING([for python_prefix-given $am_display_PYTHON exec_prefix])
+    AC_MSG_RESULT([$am_cv_python_exec_prefix])],
+   [
+    # Set am__usable_exec_prefix whether using GNU or Python values,
+    # since we use that variable for pyexecdir.
+    if test "x$exec_prefix" = xNONE; then
+      am__usable_exec_prefix=$am__usable_prefix
+    else
+      am__usable_exec_prefix=$exec_prefix
+    fi
+    #
+    if $am_use_python_sys; then # using python sys.exec_prefix, not GNU
+      AC_CACHE_CHECK([for python default $am_display_PYTHON exec_prefix],
+      [am_cv_python_exec_prefix],
+      [am_cv_python_exec_prefix=`$PYTHON -c "import sys; sys.stdout.write(sys.exec_prefix)"`])
+      dnl If sys.exec_prefix is a subdir of $exec_prefix, replace the
+      dnl literal value of $exec_prefix with a variable reference so it can
+      dnl be overridden.
+      case $am_cv_python_exec_prefix in
+      $am__usable_exec_prefix*)
+        am__strip_prefix=`echo "$am__usable_exec_prefix" | sed 's|.|.|g'`
+        am_python_exec_prefix_subst=`echo "$am_cv_python_exec_prefix" | sed "s,^$am__strip_prefix,\\${exec_prefix},"`
+        ;;
+      *)
+        am_python_exec_prefix_subst=$am_cv_python_exec_prefix
+        ;;
+     esac
+   else # using GNU $exec_prefix, not python sys.exec_prefix
+     am_python_exec_prefix_subst='${exec_prefix}'
+     am_python_exec_prefix=$am_python_exec_prefix_subst
+     AC_MSG_CHECKING([for GNU default $am_display_PYTHON exec_prefix])
+     AC_MSG_RESULT([$am_python_exec_prefix])
+   fi])])
+  # Substituting python_exec_prefix_subst.
+  AC_SUBST([PYTHON_EXEC_PREFIX], [$am_python_exec_prefix_subst])
+
+  # Factor out some code duplication into this shell variable.
   am_python_setup_sysconfig="\
 import sys
 # Prefer sysconfig over distutils.sysconfig, for better compatibility
@@ -541,96 +646,95 @@ try:
 except ImportError:
     pass"
 
-  dnl Set up 4 directories:
+  dnl emacs-page Set up 4 directories:
 
-  dnl pythondir -- where to install python scripts.  This is the
-  dnl   site-packages directory, not the python standard library
-  dnl   directory like in previous automake betas.  This behavior
-  dnl   is more consistent with lispdir.m4 for example.
+  dnl 1. pythondir: where to install python scripts.  This is the
+  dnl    site-packages directory, not the python standard library
+  dnl    directory like in previous automake betas.  This behavior
+  dnl    is more consistent with lispdir.m4 for example.
   dnl Query distutils for this directory.
-  AC_CACHE_CHECK([for $am_display_PYTHON script directory],
-    [am_cv_python_pythondir],
-    [if test "x$prefix" = xNONE
-     then
-       am_py_prefix=$ac_default_prefix
-     else
-       am_py_prefix=$prefix
-     fi
-     am_cv_python_pythondir=`$PYTHON -c "
+  dnl
+  AC_CACHE_CHECK([for $am_display_PYTHON script directory (pythondir)],
+  [am_cv_python_pythondir],
+  [if test "x$am_cv_python_prefix" = x; then
+     am_py_prefix=$am__usable_prefix
+   else
+     am_py_prefix=$am_cv_python_prefix
+   fi
+   am_cv_python_pythondir=`$PYTHON -c "
 $am_python_setup_sysconfig
 if can_use_sysconfig:
-    sitedir = sysconfig.get_path('purelib', vars={'base':'$am_py_prefix'})
+  sitedir = sysconfig.get_path('purelib', vars={'base':'$am_py_prefix'})
 else:
-    from distutils import sysconfig
-    sitedir = sysconfig.get_python_lib(0, 0, prefix='$am_py_prefix')
+  from distutils import sysconfig
+  sitedir = sysconfig.get_python_lib(0, 0, prefix='$am_py_prefix')
 sys.stdout.write(sitedir)"`
-     case $am_cv_python_pythondir in
-     $am_py_prefix*)
-       am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
-       am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,$PYTHON_PREFIX,"`
-       ;;
-     *)
-       case $am_py_prefix in
-         /usr|/System*) ;;
-         *)
-	  am_cv_python_pythondir=$PYTHON_PREFIX/lib/python$PYTHON_VERSION/site-packages
-	  ;;
-       esac
-       ;;
+   #
+   case $am_cv_python_pythondir in
+   $am_py_prefix*)
+     am__strip_prefix=`echo "$am_py_prefix" | sed 's|.|.|g'`
+     am_cv_python_pythondir=`echo "$am_cv_python_pythondir" | sed "s,^$am__strip_prefix,\\${PYTHON_PREFIX},"`
+     ;;
+   *)
+     case $am_py_prefix in
+       /usr|/System*) ;;
+       *) am_cv_python_pythondir="\${PYTHON_PREFIX}/lib/python$PYTHON_VERSION/site-packages"
+          ;;
      esac
-    ])
+     ;;
+   esac
+  ])
   AC_SUBST([pythondir], [$am_cv_python_pythondir])
 
-  dnl pkgpythondir -- $PACKAGE directory under pythondir.  Was
-  dnl   PYTHON_SITE_PACKAGE in previous betas, but this naming is
-  dnl   more consistent with the rest of automake.
-
+  dnl 2. pkgpythondir: $PACKAGE directory under pythondir.  Was
+  dnl    PYTHON_SITE_PACKAGE in previous betas, but this naming is
+  dnl    more consistent with the rest of automake.
+  dnl
   AC_SUBST([pkgpythondir], [\${pythondir}/$PACKAGE])
 
-  dnl pyexecdir -- directory for installing python extension modules
-  dnl   (shared libraries)
+  dnl 3. pyexecdir: directory for installing python extension modules
+  dnl    (shared libraries).
   dnl Query distutils for this directory.
-  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory],
-    [am_cv_python_pyexecdir],
-    [if test "x$exec_prefix" = xNONE
-     then
-       am_py_exec_prefix=$am_py_prefix
-     else
-       am_py_exec_prefix=$exec_prefix
-     fi
-     am_cv_python_pyexecdir=`$PYTHON -c "
+  dnl
+  AC_CACHE_CHECK([for $am_display_PYTHON extension module directory (pyexecdir)],
+  [am_cv_python_pyexecdir],
+  [if test "x$am_cv_python_exec_prefix" = x; then
+     am_py_exec_prefix=$am__usable_exec_prefix
+   else
+     am_py_exec_prefix=$am_cv_python_exec_prefix
+   fi
+   am_cv_python_pyexecdir=`$PYTHON -c "
 $am_python_setup_sysconfig
 if can_use_sysconfig:
-    sitedir = sysconfig.get_path('platlib', vars={'platbase':'$am_py_prefix'})
+  sitedir = sysconfig.get_path('platlib', vars={'platbase':'$am_py_exec_prefix'})
 else:
-    from distutils import sysconfig
-    sitedir = sysconfig.get_python_lib(1, 0, prefix='$am_py_prefix')
+  from distutils import sysconfig
+  sitedir = sysconfig.get_python_lib(1, 0, prefix='$am_py_exec_prefix')
 sys.stdout.write(sitedir)"`
-     case $am_cv_python_pyexecdir in
-     $am_py_exec_prefix*)
-       am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
-       am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,$PYTHON_EXEC_PREFIX,"`
-       ;;
-     *)
-       case $am_py_exec_prefix in
-         /usr|/System*) ;;
-         *)
-	   am_cv_python_pyexecdir=$PYTHON_EXEC_PREFIX/lib/python$PYTHON_VERSION/site-packages
-	   ;;
-       esac
-       ;;
+   #
+   case $am_cv_python_pyexecdir in
+   $am_py_exec_prefix*)
+     am__strip_prefix=`echo "$am_py_exec_prefix" | sed 's|.|.|g'`
+     am_cv_python_pyexecdir=`echo "$am_cv_python_pyexecdir" | sed "s,^$am__strip_prefix,\\${PYTHON_EXEC_PREFIX},"`
+     ;;
+   *)
+     case $am_py_exec_prefix in
+       /usr|/System*) ;;
+       *) am_cv_python_pyexecdir="\${PYTHON_EXEC_PREFIX}/lib/python$PYTHON_VERSION/site-packages"
+          ;;
      esac
-    ])
+     ;;
+   esac
+  ])
   AC_SUBST([pyexecdir], [$am_cv_python_pyexecdir])
 
-  dnl pkgpyexecdir -- $(pyexecdir)/$(PACKAGE)
-
+  dnl 4. pkgpyexecdir: $(pyexecdir)/$(PACKAGE)
+  dnl
   AC_SUBST([pkgpyexecdir], [\${pyexecdir}/$PACKAGE])
 
   dnl Run any user-specified action.
   $2
   fi
-
 ])
 
 
@@ -653,7 +757,7 @@ for i in list(range(0, 4)): minverhex = (minverhex << 8) + minver[[i]]
 sys.exit(sys.hexversion < minverhex)"
   AS_IF([AM_RUN_LOG([$1 -c "$prog"])], [$3], [$4])])
 
-# Copyright (C) 2001-2020 Free Software Foundation, Inc.
+# Copyright (C) 2001-2021 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,

autoconf/install-sh

@@ -1,7 +1,7 @@
-#!/bin/sh
+#!/usr/bin/sh
 # install - install a program, script, or datafile
 
-scriptversion=2006-10-14.15
+scriptversion=2020-11-14.01; # UTC
 
 # This originates from X11R5 (mit/util/scripts/install.sh), which was
 # later released in X11R6 (xc/config/util/install.sh) with the
@@ -35,57 +35,62 @@ scriptversion=2006-10-14.15
 # FSF changes to this file are in the public domain.
 #
 # Calling this script install-sh is preferred over install.sh, to prevent
-# `make' implicit rules from creating a file called install from it
+# 'make' implicit rules from creating a file called install from it
 # when there is no Makefile.
 #
 # This script is compatible with the BSD install script, but was written
 # from scratch.
 
+tab='	'
 nl='
 '
-IFS=" ""	$nl"
+IFS=" $tab$nl"
 
-# set DOITPROG to echo to test this script
+# Set DOITPROG to "echo" to test this script.
 
-# Don't use :- since 4.3BSD and earlier shells don't like it.
-doit="${DOITPROG-}"
-if test -z "$doit"; then
-  doit_exec=exec
-else
-  doit_exec=$doit
-fi
+doit=${DOITPROG-}
+doit_exec=${doit:-exec}
 
 # Put in absolute file names if you don't have them in your path;
 # or use environment vars.
 
-mvprog="${MVPROG-mv}"
-cpprog="${CPPROG-cp}"
-chmodprog="${CHMODPROG-chmod}"
-chownprog="${CHOWNPROG-chown}"
-chgrpprog="${CHGRPPROG-chgrp}"
-stripprog="${STRIPPROG-strip}"
-rmprog="${RMPROG-rm}"
-mkdirprog="${MKDIRPROG-mkdir}"
+chgrpprog=${CHGRPPROG-chgrp}
+chmodprog=${CHMODPROG-chmod}
+chownprog=${CHOWNPROG-chown}
+cmpprog=${CMPPROG-cmp}
+cpprog=${CPPROG-cp}
+mkdirprog=${MKDIRPROG-mkdir}
+mvprog=${MVPROG-mv}
+rmprog=${RMPROG-rm}
+stripprog=${STRIPPROG-strip}
 
-posix_glob=
 posix_mkdir=
 
 # Desired mode of installed file.
 mode=0755
 
+# Create dirs (including intermediate dirs) using mode 755.
+# This is like GNU 'install' as of coreutils 8.32 (2020).
+mkdir_umask=22
+
+backupsuffix=
+chgrpcmd=
 chmodcmd=$chmodprog
 chowncmd=
-chgrpcmd=
-stripcmd=
+mvcmd=$mvprog
 rmcmd="$rmprog -f"
-mvcmd="$mvprog"
+stripcmd=
+
 src=
 dst=
 dir_arg=
-dstarg=
-no_target_directory=
+dst_arg=
 
-usage="Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
+copy_on_change=false
+is_target_a_directory=possibly
+
+usage="\
+Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
    or: $0 [OPTION]... SRCFILES... DIRECTORY
    or: $0 [OPTION]... -t DIRECTORY SRCFILES...
    or: $0 [OPTION]... -d DIRECTORIES...
@@ -95,91 +100,116 @@ In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
 In the 4th, create DIRECTORIES.
 
 Options:
--c         (ignored)
--d         create directories instead of installing files.
--g GROUP   $chgrpprog installed files to GROUP.
--m MODE    $chmodprog installed files to MODE.
--o USER    $chownprog installed files to USER.
--s         $stripprog installed files.
--t DIRECTORY  install into DIRECTORY.
--T         report an error if DSTFILE is a directory.
---help     display this help and exit.
---version  display version info and exit.
+     --help     display this help and exit.
+     --version  display version info and exit.
+
+  -c            (ignored)
+  -C            install only if different (preserve data modification time)
+  -d            create directories instead of installing files.
+  -g GROUP      $chgrpprog installed files to GROUP.
+  -m MODE       $chmodprog installed files to MODE.
+  -o USER       $chownprog installed files to USER.
+  -p            pass -p to $cpprog.
+  -s            $stripprog installed files.
+  -S SUFFIX     attempt to back up existing files, with suffix SUFFIX.
+  -t DIRECTORY  install into DIRECTORY.
+  -T            report an error if DSTFILE is a directory.
 
 Environment variables override the default commands:
-  CHGRPPROG CHMODPROG CHOWNPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG
+  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
+  RMPROG STRIPPROG
+
+By default, rm is invoked with -f; when overridden with RMPROG,
+it's up to you to specify -f if you want it.
+
+If -S is not specified, no backups are attempted.
+
+Email bug reports to bug-automake@gnu.org.
+Automake home page: https://www.gnu.org/software/automake/
 "
 
 while test $# -ne 0; do
   case $1 in
-    -c) shift
-        continue;;
+    -c) ;;
 
-    -d) dir_arg=true
-        shift
-        continue;;
+    -C) copy_on_change=true;;
+
+    -d) dir_arg=true;;
 
     -g) chgrpcmd="$chgrpprog $2"
-        shift
-        shift
-        continue;;
+        shift;;
 
     --help) echo "$usage"; exit $?;;
 
     -m) mode=$2
-        shift
-        shift
-	case $mode in
-	  *' '* | *'	'* | *'
-'*	  | *'*'* | *'?'* | *'['*)
-	    echo "$0: invalid mode: $mode" >&2
-	    exit 1;;
-	esac
-        continue;;
+        case $mode in
+          *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*)
+            echo "$0: invalid mode: $mode" >&2
+            exit 1;;
+        esac
+        shift;;
 
     -o) chowncmd="$chownprog $2"
-        shift
-        shift
-        continue;;
+        shift;;
 
-    -s) stripcmd=$stripprog
-        shift
-        continue;;
+    -p) cpprog="$cpprog -p";;
 
-    -t) dstarg=$2
-	shift
-	shift
-	continue;;
+    -s) stripcmd=$stripprog;;
 
-    -T) no_target_directory=true
-	shift
-	continue;;
+    -S) backupsuffix="$2"
+        shift;;
+
+    -t)
+        is_target_a_directory=always
+        dst_arg=$2
+        # Protect names problematic for 'test' and other utilities.
+        case $dst_arg in
+          -* | [=\(\)!]) dst_arg=./$dst_arg;;
+        esac
+        shift;;
+
+    -T) is_target_a_directory=never;;
 
     --version) echo "$0 $scriptversion"; exit $?;;
 
-    --)	shift
-	break;;
+    --) shift
+        break;;
 
-    -*)	echo "$0: invalid option: $1" >&2
-	exit 1;;
+    -*) echo "$0: invalid option: $1" >&2
+        exit 1;;
 
     *)  break;;
   esac
+  shift
 done
 
-if test $# -ne 0 && test -z "$dir_arg$dstarg"; then
+# We allow the use of options -d and -T together, by making -d
+# take the precedence; this is for compatibility with GNU install.
+
+if test -n "$dir_arg"; then
+  if test -n "$dst_arg"; then
+    echo "$0: target directory not allowed when installing a directory." >&2
+    exit 1
+  fi
+fi
+
+if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
   # When -d is used, all remaining arguments are directories to create.
   # When -t is used, the destination is already specified.
   # Otherwise, the last argument is the destination.  Remove it from $@.
   for arg
   do
-    if test -n "$dstarg"; then
+    if test -n "$dst_arg"; then
       # $@ is not empty: it contains at least $arg.
-      set fnord "$@" "$dstarg"
+      set fnord "$@" "$dst_arg"
       shift # fnord
     fi
     shift # arg
-    dstarg=$arg
+    dst_arg=$arg
+    # Protect names problematic for 'test' and other utilities.
+    case $dst_arg in
+      -* | [=\(\)!]) dst_arg=./$dst_arg;;
+    esac
   done
 fi
 
@@ -188,13 +218,26 @@ if test $# -eq 0; then
     echo "$0: no input file specified." >&2
     exit 1
   fi
-  # It's OK to call `install-sh -d' without argument.
+  # It's OK to call 'install-sh -d' without argument.
   # This can happen when creating conditional directories.
   exit 0
 fi
 
 if test -z "$dir_arg"; then
-  trap '(exit $?); exit' 1 2 13 15
+  if test $# -gt 1 || test "$is_target_a_directory" = always; then
+    if test ! -d "$dst_arg"; then
+      echo "$0: $dst_arg: Is not a directory." >&2
+      exit 1
+    fi
+  fi
+fi
+
+if test -z "$dir_arg"; then
+  do_exit='(exit $ret); exit $ret'
+  trap "ret=129; $do_exit" 1
+  trap "ret=130; $do_exit" 2
+  trap "ret=141; $do_exit" 13
+  trap "ret=143; $do_exit" 15
 
   # Set umask so as not to create temps with too-generous modes.
   # However, 'strip' requires both read and write access to temps.
@@ -205,16 +248,16 @@ if test -z "$dir_arg"; then
 
     *[0-7])
       if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
       else
-	u_plus_rw='% 200'
+        u_plus_rw='% 200'
       fi
       cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
     *)
       if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
       else
-	u_plus_rw=,u+rw
+        u_plus_rw=,u+rw
       fi
       cp_umask=$mode$u_plus_rw;;
   esac
@@ -222,9 +265,9 @@ fi
 
 for src
 do
-  # Protect names starting with `-'.
+  # Protect names problematic for 'test' and other utilities.
   case $src in
-    -*) src=./$src ;;
+    -* | [=\(\)!]) src=./$src;;
   esac
 
   if test -n "$dir_arg"; then
@@ -232,6 +275,10 @@ do
     dstdir=$dst
     test -d "$dstdir"
     dstdir_status=$?
+    # Don't chown directories that already exist.
+    if test $dstdir_status = 0; then
+      chowncmd=""
+    fi
   else
 
     # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
@@ -242,196 +289,154 @@ do
       exit 1
     fi
 
-    if test -z "$dstarg"; then
+    if test -z "$dst_arg"; then
       echo "$0: no destination specified." >&2
       exit 1
     fi
+    dst=$dst_arg
 
-    dst=$dstarg
-    # Protect names starting with `-'.
-    case $dst in
-      -*) dst=./$dst ;;
-    esac
-
-    # If destination is a directory, append the input filename; won't work
-    # if double slashes aren't ignored.
+    # If destination is a directory, append the input filename.
     if test -d "$dst"; then
-      if test -n "$no_target_directory"; then
-	echo "$0: $dstarg: Is a directory" >&2
-	exit 1
+      if test "$is_target_a_directory" = never; then
+        echo "$0: $dst_arg: Is a directory" >&2
+        exit 1
       fi
       dstdir=$dst
-      dst=$dstdir/`basename "$src"`
+      dstbase=`basename "$src"`
+      case $dst in
+	*/) dst=$dst$dstbase;;
+	*)  dst=$dst/$dstbase;;
+      esac
       dstdir_status=0
     else
-      # Prefer dirname, but fall back on a substitute if dirname fails.
-      dstdir=`
-	(dirname "$dst") 2>/dev/null ||
-	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
-	     X"$dst" : 'X\(//\)[^/]' \| \
-	     X"$dst" : 'X\(//\)$' \| \
-	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
-	echo X"$dst" |
-	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)[^/].*/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\).*/{
-		   s//\1/
-		   q
-		 }
-		 s/.*/./; q'
-      `
-
+      dstdir=`dirname "$dst"`
       test -d "$dstdir"
       dstdir_status=$?
     fi
   fi
 
+  case $dstdir in
+    */) dstdirslash=$dstdir;;
+    *)  dstdirslash=$dstdir/;;
+  esac
+
   obsolete_mkdir_used=false
 
   if test $dstdir_status != 0; then
     case $posix_mkdir in
       '')
-	# Create intermediate dirs using mode 755 as modified by the umask.
-	# This is like FreeBSD 'install' as of 1997-10-28.
-	umask=`umask`
-	case $stripcmd.$umask in
-	  # Optimize common cases.
-	  *[2367][2367]) mkdir_umask=$umask;;
-	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+        # With -d, create the new directory with the user-specified mode.
+        # Otherwise, rely on $mkdir_umask.
+        if test -n "$dir_arg"; then
+          mkdir_mode=-m$mode
+        else
+          mkdir_mode=
+        fi
 
-	  *[0-7])
-	    mkdir_umask=`expr $umask + 22 \
-	      - $umask % 100 % 40 + $umask % 20 \
-	      - $umask % 10 % 4 + $umask % 2
-	    `;;
-	  *) mkdir_umask=$umask,go-w;;
-	esac
+        posix_mkdir=false
+	# The $RANDOM variable is not portable (e.g., dash).  Use it
+	# here however when possible just to lower collision chance.
+	tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
 
-	# With -d, create the new directory with the user-specified mode.
-	# Otherwise, rely on $mkdir_umask.
-	if test -n "$dir_arg"; then
-	  mkdir_mode=-m$mode
+	trap '
+	  ret=$?
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null
+	  exit $ret
+	' 0
+
+	# Because "mkdir -p" follows existing symlinks and we likely work
+	# directly in world-writeable /tmp, make sure that the '$tmpdir'
+	# directory is successfully created first before we actually test
+	# 'mkdir -p'.
+	if (umask $mkdir_umask &&
+	    $mkdirprog $mkdir_mode "$tmpdir" &&
+	    exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1
+	then
+	  if test -z "$dir_arg" || {
+	       # Check for POSIX incompatibilities with -m.
+	       # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+	       # other-writable bit of parent directory when it shouldn't.
+	       # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+	       test_tmpdir="$tmpdir/a"
+	       ls_ld_tmpdir=`ls -ld "$test_tmpdir"`
+	       case $ls_ld_tmpdir in
+		 d????-?r-*) different_mode=700;;
+		 d????-?--*) different_mode=755;;
+		 *) false;;
+	       esac &&
+	       $mkdirprog -m$different_mode -p -- "$test_tmpdir" && {
+		 ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"`
+		 test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+	       }
+	     }
+	  then posix_mkdir=:
+	  fi
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir"
 	else
-	  mkdir_mode=
+	  # Remove any dirs left behind by ancient mkdir implementations.
+	  rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null
 	fi
-
-	posix_mkdir=false
-	case $umask in
-	  *[123567][0-7][0-7])
-	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
-	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
-	    ;;
-	  *)
-	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
-	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
-	    if (umask $mkdir_umask &&
-		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
-	    then
-	      if test -z "$dir_arg" || {
-		   # Check for POSIX incompatibilities with -m.
-		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
-		   # other-writeable bit of parent directory when it shouldn't.
-		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
-		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
-		   case $ls_ld_tmpdir in
-		     d????-?r-*) different_mode=700;;
-		     d????-?--*) different_mode=755;;
-		     *) false;;
-		   esac &&
-		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
-		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
-		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
-		   }
-		 }
-	      then posix_mkdir=:
-	      fi
-	      rmdir "$tmpdir/d" "$tmpdir"
-	    else
-	      # Remove any dirs left behind by ancient mkdir implementations.
-	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
-	    fi
-	    trap '' 0;;
-	esac;;
+	trap '' 0;;
     esac
 
     if
       $posix_mkdir && (
-	umask $mkdir_umask &&
-	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+        umask $mkdir_umask &&
+        $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
       )
     then :
     else
 
-      # The umask is ridiculous, or mkdir does not conform to POSIX,
+      # mkdir does not conform to POSIX,
       # or it failed possibly due to a race condition.  Create the
       # directory the slow way, step by step, checking for races as we go.
 
       case $dstdir in
-	/*) prefix=/ ;;
-	-*) prefix=./ ;;
-	*)  prefix= ;;
-      esac
-
-      case $posix_glob in
-        '')
-	  if (set -f) 2>/dev/null; then
-	    posix_glob=true
-	  else
-	    posix_glob=false
-	  fi ;;
+        /*) prefix='/';;
+        [-=\(\)!]*) prefix='./';;
+        *)  prefix='';;
       esac
 
       oIFS=$IFS
       IFS=/
-      $posix_glob && set -f
+      set -f
       set fnord $dstdir
       shift
-      $posix_glob && set +f
+      set +f
       IFS=$oIFS
 
       prefixes=
 
       for d
       do
-	test -z "$d" && continue
+        test X"$d" = X && continue
 
-	prefix=$prefix$d
-	if test -d "$prefix"; then
-	  prefixes=
-	else
-	  if $posix_mkdir; then
-	    (umask=$mkdir_umask &&
-	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
-	    # Don't fail if two instances are running concurrently.
-	    test -d "$prefix" || exit 1
-	  else
-	    case $prefix in
-	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
-	      *) qprefix=$prefix;;
-	    esac
-	    prefixes="$prefixes '$qprefix'"
-	  fi
-	fi
-	prefix=$prefix/
+        prefix=$prefix$d
+        if test -d "$prefix"; then
+          prefixes=
+        else
+          if $posix_mkdir; then
+            (umask $mkdir_umask &&
+             $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+            # Don't fail if two instances are running concurrently.
+            test -d "$prefix" || exit 1
+          else
+            case $prefix in
+              *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+              *) qprefix=$prefix;;
+            esac
+            prefixes="$prefixes '$qprefix'"
+          fi
+        fi
+        prefix=$prefix/
       done
 
       if test -n "$prefixes"; then
-	# Don't fail if two instances are running concurrently.
-	(umask $mkdir_umask &&
-	 eval "\$doit_exec \$mkdirprog $prefixes") ||
-	  test -d "$dstdir" || exit 1
-	obsolete_mkdir_used=true
+        # Don't fail if two instances are running concurrently.
+        (umask $mkdir_umask &&
+         eval "\$doit_exec \$mkdirprog $prefixes") ||
+          test -d "$dstdir" || exit 1
+        obsolete_mkdir_used=true
       fi
     fi
   fi
@@ -444,14 +449,25 @@ do
   else
 
     # Make a couple of temp file names in the proper directory.
-    dsttmp=$dstdir/_inst.$$_
-    rmtmp=$dstdir/_rm.$$_
+    dsttmp=${dstdirslash}_inst.$$_
+    rmtmp=${dstdirslash}_rm.$$_
 
     # Trap to clean up those temp files at exit.
     trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
 
     # Copy the file name to the temp name.
-    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
+    (umask $cp_umask &&
+     { test -z "$stripcmd" || {
+	 # Create $dsttmp read-write so that cp doesn't create it read-only,
+	 # which would cause strip to fail.
+	 if test -z "$doit"; then
+	   : >"$dsttmp" # No need to fork-exec 'touch'.
+	 else
+	   $doit touch "$dsttmp"
+	 fi
+       }
+     } &&
+     $doit_exec $cpprog "$src" "$dsttmp") &&
 
     # and set any options; do chmod last to preserve setuid bits.
     #
@@ -459,49 +475,67 @@ do
     # ignore errors from any of these, just make sure not to ignore
     # errors from the above "$doit $cpprog $src $dsttmp" command.
     #
-    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } \
-      && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } \
-      && { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } \
-      && { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
+    { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
+    { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
+    { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
+    { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
 
-    # Now rename the file to the real destination.
-    { $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null \
-      || {
-	   # The rename failed, perhaps because mv can't rename something else
-	   # to itself, or perhaps because mv is so ancient that it does not
-	   # support -f.
+    # If -C, don't bother to copy if it wouldn't change the file.
+    if $copy_on_change &&
+       old=`LC_ALL=C ls -dlL "$dst"     2>/dev/null` &&
+       new=`LC_ALL=C ls -dlL "$dsttmp"  2>/dev/null` &&
+       set -f &&
+       set X $old && old=:$2:$4:$5:$6 &&
+       set X $new && new=:$2:$4:$5:$6 &&
+       set +f &&
+       test "$old" = "$new" &&
+       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
+    then
+      rm -f "$dsttmp"
+    else
+      # If $backupsuffix is set, and the file being installed
+      # already exists, attempt a backup.  Don't worry if it fails,
+      # e.g., if mv doesn't support -f.
+      if test -n "$backupsuffix" && test -f "$dst"; then
+        $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null
+      fi
 
-	   # Now remove or move aside any old file at destination location.
-	   # We try this two ways since rm can't unlink itself on some
-	   # systems and the destination file might be busy for other
-	   # reasons.  In this case, the final cleanup might fail but the new
-	   # file should still install successfully.
-	   {
-	     if test -f "$dst"; then
-	       $doit $rmcmd -f "$dst" 2>/dev/null \
-	       || { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null \
-		     && { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }; }\
-	       || {
-		 echo "$0: cannot unlink or rename $dst" >&2
-		 (exit 1); exit 1
-	       }
-	     else
-	       :
-	     fi
-	   } &&
+      # Rename the file to the real destination.
+      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
 
-	   # Now rename the file to the real destination.
-	   $doit $mvcmd "$dsttmp" "$dst"
-	 }
-    } || exit 1
+      # The rename failed, perhaps because mv can't rename something else
+      # to itself, or perhaps because mv is so ancient that it does not
+      # support -f.
+      {
+        # Now remove or move aside any old file at destination location.
+        # We try this two ways since rm can't unlink itself on some
+        # systems and the destination file might be busy for other
+        # reasons.  In this case, the final cleanup might fail but the new
+        # file should still install successfully.
+        {
+          test ! -f "$dst" ||
+          $doit $rmcmd "$dst" 2>/dev/null ||
+          { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+            { $doit $rmcmd "$rmtmp" 2>/dev/null; :; }
+          } ||
+          { echo "$0: cannot unlink or rename $dst" >&2
+            (exit 1); exit 1
+          }
+        } &&
+
+        # Now rename the file to the real destination.
+        $doit $mvcmd "$dsttmp" "$dst"
+      }
+    fi || exit 1
 
     trap '' 0
   fi
 done
 
 # Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
+# eval: (add-hook 'before-save-hook 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-end: "$"
+# time-stamp-time-zone: "UTC0"
+# time-stamp-end: "; # UTC"
 # End:

conf/example.conf.in

@@ -126,7 +126,7 @@ devices {
 	# be used, regardless of this setting, when the --devicesfile
 	# option is set to a specific file name.
 	# This configuration option has an automatic default value.
-	# use_devicesfile = 0
+	# use_devicesfile = @DEFAULT_USE_DEVICES_FILE@
 
 	# Configuration option devices/devicesfile.
 	# The name of the system devices file, listing devices that LVM should use.
@@ -625,13 +625,12 @@ allocation {
 	# Enables or disables whether VDO volume should tag its latency-critical
 	# writes with the REQ_SYNC flag. Some device mapper targets such as dm-raid5
 	# process writes with this flag at a higher priority.
-	# Default is enabled.
 	# This configuration option has an automatic default value.
 	# vdo_use_metadata_hints = 1
 
 	# Configuration option allocation/vdo_minimum_io_size.
 	# The minimum IO size for VDO volume to accept, in bytes.
-	# Valid values are 512 or 4096. The recommended and default value is 4096.
+	# Valid values are 512 or 4096. The recommended value is 4096.
 	# This configuration option has an automatic default value.
 	# vdo_minimum_io_size = 4096
 
@@ -684,7 +683,7 @@ allocation {
 	# Configuration option allocation/vdo_bio_threads.
 	# Specifies the number of threads to use for submitting I/O
 	# operations to the storage device of VDO volume.
-	# The value must be in range [1..100]
+	# The value must be in range [1..100].
 	# Each additional thread after the first will use an additional 18MiB of RAM,
 	# plus 1.12 MiB of RAM per megabyte of configured read cache size.
 	# This configuration option has an automatic default value.
@@ -698,7 +697,7 @@ allocation {
 
 	# Configuration option allocation/vdo_cpu_threads.
 	# Specifies the number of threads to use for CPU-intensive work such as
-	# hashing or compression for VDO volume. The value must be in range [1..100]
+	# hashing or compression for VDO volume. The value must be in range [1..100].
 	# This configuration option has an automatic default value.
 	# vdo_cpu_threads = 2
 
@@ -716,7 +715,7 @@ allocation {
 	# processing based on the hash value computed from the block data.
 	# A logical thread count of 9 or more will require explicitly specifying
 	# a sufficiently large block map cache size, as well.
-	# The value must be in range [0..100].
+	# The value must be in range [0..60].
 	# vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be
 	# either all zero or all non-zero.
 	# This configuration option has an automatic default value.
@@ -758,7 +757,7 @@ allocation {
 	# vdo_max_discard = 1
 
 	# Configuration option allocation/vdo_pool_header_size.
-	# Specified the emptry header size in KiB at the front and end of vdo pool device.
+	# Specified the empty header size in KiB at the front and end of vdo pool device.
 	# This configuration option has an automatic default value.
 	# vdo_pool_header_size = 512
 }
@@ -937,7 +936,7 @@ backup {
 	# archive = 1
 
 	# Configuration option backup/archive_dir.
-	# Location of the metdata archive files.
+	# Location of the metadata archive files.
 	# Remember to back up this directory regularly!
 	# This configuration option has an automatic default value.
 	# archive_dir = "@DEFAULT_SYS_DIR@/@DEFAULT_ARCHIVE_SUBDIR@"
@@ -1464,13 +1463,13 @@ activation {
 
 	# Configuration option activation/reserved_stack.
 	# Stack size in KiB to reserve for use while devices are suspended.
-	# Insufficent reserve risks I/O deadlock during device suspension.
+	# Insufficient reserve risks I/O deadlock during device suspension.
 	# This configuration option has an automatic default value.
 	# reserved_stack = 64
 
 	# Configuration option activation/reserved_memory.
 	# Memory size in KiB to reserve for use while devices are suspended.
-	# Insufficent reserve risks I/O deadlock during device suspension.
+	# Insufficient reserve risks I/O deadlock during device suspension.
 	# This configuration option has an automatic default value.
 	# reserved_memory = 8192
 
@@ -1605,7 +1604,7 @@ activation {
 	# This includes LVs that have the following segment types:
 	# raid1, raid4, raid5*, and raid6*.
 	# If a device in the LV fails, the policy determines the steps
-	# performed by dmeventd automatically, and the steps perfomed by the
+	# performed by dmeventd automatically, and the steps performed by the
 	# manual command lvconvert --repair --use-policies.
 	# Automatic handling requires dmeventd to be monitoring the LV.
 	#
@@ -1629,7 +1628,7 @@ activation {
 	# (copies) and a mirror log. A disk log ensures that a mirror LV does
 	# not need to be re-synced (all copies made the same) every time a
 	# machine reboots or crashes. If a device in the LV fails, this policy
-	# determines the steps perfomed by dmeventd automatically, and the steps
+	# determines the steps performed by dmeventd automatically, and the steps
 	# performed by the manual command lvconvert --repair --use-policies.
 	# Automatic handling requires dmeventd to be monitoring the LV.
 	#
@@ -1952,6 +1951,12 @@ activation {
 	#     name for identification.
 	#   json
 	#     JSON format.
+	#   json_std
+	#     JSON format that is more compliant with JSON standard.
+	#     Compared to original "json" format:
+	#       - it does not use double quotes around numeric values,
+	#       - it uses 'null' for undefined numeric values,
+	#       - it prints string list as proper JSON array of strings instead of a single string.
 	# This configuration option has an automatic default value.
 	# output_format = "basic"
 

configure.ac

@@ -83,6 +83,7 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 AC_PROG_MKDIR_P
 AC_PROG_RANLIB
+AC_CHECK_TOOL([READELF], [readelf])
 AC_CHECK_TOOL(AR, ar)
 AC_PATH_TOOL(CFLOW_CMD, cflow)
 AC_PATH_TOOL(CSCOPE_CMD, cscope)
@@ -95,9 +96,6 @@ dnl -- Check for header files.
 AC_HEADER_DIRENT
 AC_HEADER_MAJOR
 AC_HEADER_STDBOOL
-AC_HEADER_STDC
-AC_HEADER_SYS_WAIT
-AC_HEADER_TIME
 
 AC_CHECK_HEADERS([assert.h ctype.h dirent.h errno.h fcntl.h float.h \
   getopt.h inttypes.h langinfo.h libgen.h limits.h locale.h paths.h \
@@ -125,7 +123,6 @@ AC_STRUCT_ST_BLOCKS
 AC_STRUCT_TM
 AC_TYPE_OFF_T
 AC_TYPE_PID_T
-AC_TYPE_SIGNAL
 AC_TYPE_SIZE_T
 AC_TYPE_MODE_T
 AC_TYPE_INT8_T
@@ -153,7 +150,7 @@ AC_CHECK_FUNCS([ftruncate gethostname getpagesize gettimeofday localtime_r \
   memchr memset mkdir mkfifo munmap nl_langinfo pselect realpath rmdir setenv \
   setlocale strcasecmp strchr strcspn strdup strerror strncasecmp strndup \
   strrchr strspn strstr strtol strtoul uname], , [AC_MSG_ERROR(bailing out)])
-AC_CHECK_FUNCS([ffs prlimit versionsort])
+AC_CHECK_FUNCS([ffs mallinfo2 prlimit versionsort])
 AC_FUNC_ALLOCA
 AC_FUNC_CLOSEDIR_VOID
 AC_FUNC_CHOWN
@@ -293,7 +290,7 @@ case "$DEFAULT_USE_DEVICES_FILE" in
 esac
 AC_MSG_RESULT($DEFAULT_USE_DEVICES_FILE)
 AC_DEFINE_UNQUOTED(DEFAULT_USE_DEVICES_FILE, [$DEFAULT_USE_DEVICES_FILE],
-		   [Default for lvm.conf use_devicefile.])
+		   [Default for lvm.conf use_devicesfile.])
 
 AC_MSG_CHECKING(default name mangling)
 AC_ARG_WITH(default-name-mangling,
@@ -1139,7 +1136,7 @@ AC_CHECK_HEADER([systemd/sd-id128.h],
 		[APP_MACHINEID_SUPPORT=no])
 AC_MSG_CHECKING(whether to support systemd appmachineid)
 AC_ARG_ENABLE(app-machineid,
-	      AC_HELP_STRING([--disable-app-machineid],
+	      AS_HELP_STRING([--disable-app-machineid],
 			     [disable LVM system ID using app-specific machine-id]),
 	      AS_IF([test "$enableval" = yes && test "$APP_MACHINEID_SUPPORT" = no],
 		    AC_MSG_ERROR([Enabling app machineid requires systemd/sd-id128.h and systemd >= 234.]))
@@ -1323,8 +1320,8 @@ if test "$BUILD_LVMDBUSD" = yes; then
 	test "$PYTHON3_BINDINGS" = yes && PYTHON_BINDINGS=yes
 
 	# To get this macro, install autoconf-archive package then run autoreconf
-	AC_PYTHON_MODULE([pyudev], [Required], python3)
-	AC_PYTHON_MODULE([dbus], [Required], python3)
+	AX_PYTHON_MODULE([pyudev], [Required], python3)
+	AX_PYTHON_MODULE([dbus], [Required], python3)
 fi
 
 ################################################################################
@@ -1609,6 +1606,11 @@ AC_ARG_WITH(usrsbindir,
 			   [usrsbin executables in DIR [PREFIX/sbin]]),
 	    usrsbindir=$withval, usrsbindir='${prefix}/sbin')
 
+AC_ARG_WITH(libexecdir,
+	    AS_HELP_STRING([--with-libexecdir=DIR],
+			   [libexec executables in DIR [PREFIX/libexec]]),
+	    libexecdir=$withval, libexecdir='${prefix}/libexec')
+
 ################################################################################
 AC_ARG_WITH(udev_prefix,
 	    AS_HELP_STRING([--with-udev-prefix=UPREFIX],
@@ -1713,6 +1715,11 @@ AC_DEFINE_UNQUOTED(FSADM_PATH, ["$FSADM_PATH"], [Path to fsadm binary.])
 LVMIMPORTVDO_PATH="$SBINDIR/lvm_import_vdo"
 AC_DEFINE_UNQUOTED(LVMIMPORTVDO_PATH, ["$LVMIMPORTVDO_PATH"], [Path to lvm_import_vdo script.])
 
+LIBEXECDIR="$(eval echo $(eval echo $libexecdir))"
+
+LVRESIZE_FS_HELPER_PATH="$LIBEXECDIR/lvresize_fs_helper"
+AC_DEFINE_UNQUOTED(LVRESIZE_FS_HELPER_PATH, ["$LVRESIZE_FS_HELPER_PATH"], [Path to lvresize_fs_helper script.])
+
 ################################################################################
 dnl -- dmeventd pidfile and executable path
 if test "$BUILD_DMEVENTD" = yes; then
@@ -1871,6 +1878,7 @@ AC_SUBST(DEFAULT_SPARSE_SEGTYPE)
 AC_SUBST(DEFAULT_SYS_DIR)
 AC_SUBST(DEFAULT_SYS_LOCK_DIR)
 AC_SUBST(DEFAULT_USE_BLKID_WIPING)
+AC_SUBST(DEFAULT_USE_DEVICES_FILE)
 AC_SUBST(DEFAULT_USE_LVMPOLLD)
 AC_SUBST(DEFAULT_USE_LVMLOCKD)
 AC_SUBST(DEVMAPPER)
@@ -1882,6 +1890,7 @@ AC_SUBST(DM_LIB_PATCHLEVEL)
 AC_SUBST(ELDFLAGS)
 AC_SUBST(FSADM)
 AC_SUBST(FSADM_PATH)
+AC_SUBST(LVRESIZE_FS_HELPER_PATH)
 AC_SUBST(BLKDEACTIVATE)
 AC_SUBST(HAVE_LIBDL)
 AC_SUBST(HAVE_REALTIME)
@@ -1980,6 +1989,7 @@ AC_SUBST(systemdutildir)
 AC_SUBST(tmpfilesdir)
 AC_SUBST(usrlibdir)
 AC_SUBST(usrsbindir)
+AC_SUBST(libexecdir)
 
 ################################################################################
 dnl -- First and last lines should not contain files to generate in order to

daemons/dmeventd/libdevmapper-event.c

@@ -928,8 +928,8 @@ void dm_event_log(const char *subsys, int level, const char *file,
 			start = now;
 		now -= start;
 		if (_debug_level)
-			fprintf(stream, "[%2d:%02d] %8x:%-6s%s",
-				(int)now / 60, (int)now % 60,
+			fprintf(stream, "[%2ld:%02ld] %8x:%-6s%s",
+				(long)now / 60, (long)now % 60,
 				// TODO: Maybe use shorter ID
 				// ((int)(pthread_self()) >> 6) & 0xffff,
 				(int)pthread_self(), subsys,

daemons/lvmdbusd/automatedproperties.py

@@ -88,7 +88,6 @@ class AutomatedProperties(dbus.service.Object):
 			cb, cbe, False)
 		cfg.worker_q.put(r)
 
-
 	@staticmethod
 	def _get_all_prop(obj, interface_name):
 		if interface_name in obj.interface(True):

daemons/lvmdbusd/background.py

@@ -7,16 +7,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-import subprocess
 from . import cfg
 from .cmdhandler import options_to_cli_args, LvmExecutionMeta, call_lvm
 import dbus
-from .utils import pv_range_append, pv_dest_ranges, log_error, log_debug,\
-					mt_async_call
+from .utils import pv_range_append, pv_dest_ranges, log_error, log_debug
 from .request import RequestEntry
 import threading
 import time
-import traceback
 
 
 def pv_move_lv_cmd(move_options, lv_full_name,
@@ -67,16 +64,13 @@ def _move_merge(interface_name, command, job_state):
 	# the command always as we will be getting periodic output from them on
 	# the status of the long running operation.
 
-	meta = LvmExecutionMeta(time.time(), 0, command, -1000, None, None)
-	cfg.blackbox.add(meta)
+	meta = LvmExecutionMeta(time.time(), 0, command)
+	cfg.flightrecorder.add(meta)
 
 	ec, stdout, stderr = call_lvm(command, line_cb=_move_callback,
 									cb_data=job_state)
-
-	with meta.lock:
-		meta.ended = time.time()
-		meta.ec = ec
-		meta.stderr_txt = stderr
+	ended = time.time()
+	meta.completed(ended, ec, stdout, stderr)
 
 	if ec == 0:
 		job_state.Percent = 100

daemons/lvmdbusd/cfg.py

@@ -16,6 +16,8 @@ from lvmdbusd import path
 
 LVM_CMD = os.getenv('LVM_BINARY', path.LVM_BINARY)
 
+LOCK_FILE = os.getenv("LVM_DBUSD_LOCKFILE", "/var/lock/lvm/lvmdbusd")
+
 # This is the global object manager
 om = None
 
@@ -43,6 +45,9 @@ worker_q = queue.Queue()
 # Main event loop
 loop = None
 
+# Used to instruct the daemon if we should ignore SIGTERM
+ignore_sigterm = False
+
 BUS_NAME = os.getenv('LVM_DBUS_NAME', 'com.redhat.lvmdbus1')
 BASE_INTERFACE = 'com.redhat.lvmdbus1'
 PV_INTERFACE = BASE_INTERFACE + '.Pv'
@@ -90,11 +95,14 @@ vdo_support = False
 db = None
 
 # lvm flight recorder
-blackbox = None
+flightrecorder = None
 
 # RequestEntry ctor
 create_request_entry = None
 
+# Circular debug log
+debug = None
+
 
 def exit_daemon():
     """
@@ -104,3 +112,9 @@ def exit_daemon():
     if run and loop:
         run.value = 0
         loop.quit()
+
+
+# Debug data for lvm
+lvmdebug = None
+
+systemd = False

daemons/lvmdbusd/cmdhandler.py

@@ -6,19 +6,18 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-
+import errno
 from subprocess import Popen, PIPE
 import select
 import time
 import threading
 from itertools import chain
 import collections
-import traceback
 import os
 
 from lvmdbusd import cfg
 from lvmdbusd.utils import pv_dest_ranges, log_debug, log_error, add_no_notify,\
-							make_non_block, read_decoded
+			make_non_block, read_decoded, extract_stack_trace, LvmBug, add_config_option, get_error_msg
 from lvmdbusd.lvm_shell_proxy import LVMShellProxy
 
 try:
@@ -26,7 +25,6 @@ try:
 except ImportError:
 	import json
 
-SEP = '{|}'
 
 total_time = 0.0
 total_count = 0
@@ -38,7 +36,7 @@ cmd_lock = threading.RLock()
 
 class LvmExecutionMeta(object):
 
-	def __init__(self, start, ended, cmd, ec, stdout_txt, stderr_txt):
+	def __init__(self, start, ended, cmd, ec=-1000, stdout_txt=None, stderr_txt=None):
 		self.lock = threading.RLock()
 		self.start = start
 		self.ended = ended
@@ -49,32 +47,49 @@ class LvmExecutionMeta(object):
 
 	def __str__(self):
 		with self.lock:
-			return "EC= %d for %s\n" \
-				"STARTED: %f, ENDED: %f\n" \
+			if self.ended == 0:
+				ended_txt = "still running"
+				self.ended = time.time()
+			else:
+				ended_txt = str(time.ctime(self.ended))
+
+			return 'EC= %d for "%s"\n' \
+				"STARTED: %s, ENDED: %s, DURATION: %f\n" \
 				"STDOUT=%s\n" \
 				"STDERR=%s\n" % \
-				(self.ec, str(self.cmd), self.start, self.ended, self.stdout_txt,
-				self.stderr_txt)
+				(self.ec, " ".join(self.cmd), time.ctime(self.start), ended_txt, float(self.ended) - self.start,
+					self.stdout_txt,
+					self.stderr_txt)
+
+	def completed(self, end_time, ec, stdout_txt, stderr_txt):
+		with self.lock:
+			self.ended = end_time
+			self.ec = ec
+			self.stdout_txt = stdout_txt
+			self.stderr_txt = stderr_txt
 
 
 class LvmFlightRecorder(object):
 
 	def __init__(self, size=16):
 		self.queue = collections.deque(maxlen=size)
+		self.lock = threading.RLock()
 
 	def add(self, lvm_exec_meta):
-		self.queue.append(lvm_exec_meta)
+		with self.lock:
+			self.queue.append(lvm_exec_meta)
 
 	def dump(self):
-		with cmd_lock:
+		with self.lock:
 			if len(self.queue):
-				log_error("LVM dbus flight recorder START")
+				log_error("LVM dbus flight recorder START (in order of newest to oldest)")
 				for c in reversed(self.queue):
 					log_error(str(c))
 				log_error("LVM dbus flight recorder END")
+				self.queue.clear()
 
 
-cfg.blackbox = LvmFlightRecorder()
+cfg.flightrecorder = LvmFlightRecorder()
 
 
 def _debug_c(cmd, exit_code, out):
@@ -106,6 +121,9 @@ def call_lvm(command, debug=False, line_cb=None,
 	command.insert(0, cfg.LVM_CMD)
 	command = add_no_notify(command)
 
+	# Ensure we get an error message when we fork & exec the lvm command line
+	command = add_config_option(command, "--config", 'log/command_log_selection="log_context!=''"')
+
 	process = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True,
 					env=os.environ)
 
@@ -115,7 +133,7 @@ def call_lvm(command, debug=False, line_cb=None,
 	make_non_block(process.stdout)
 	make_non_block(process.stderr)
 
-	while True:
+	while True and cfg.run.value != 0:
 		try:
 			rd_fd = [process.stdout.fileno(), process.stderr.fileno()]
 			ready = select.select(rd_fd, [], [], 2)
@@ -133,8 +151,8 @@ def call_lvm(command, debug=False, line_cb=None,
 					if i != -1:
 						try:
 							line_cb(cb_data, stdout_text[stdout_index:i])
-						except:
-							st = traceback.format_exc()
+						except BaseException as be:
+							st = extract_stack_trace(be)
 							log_error("call_lvm: line_cb exception: \n %s" % st)
 						stdout_index = i + 1
 					else:
@@ -145,12 +163,31 @@ def call_lvm(command, debug=False, line_cb=None,
 				break
 		except IOError as ioe:
 			log_debug("call_lvm:" + str(ioe))
-			pass
+			break
 
-	if debug or process.returncode != 0:
-		_debug_c(command, process.returncode, (stdout_text, stderr_text))
+	if process.returncode is not None:
+		cfg.lvmdebug.lvm_complete()
+		if debug or (process.returncode != 0 and (process.returncode != 5 and "fullreport" in command)):
+			_debug_c(command, process.returncode, (stdout_text, stderr_text))
+
+		try:
+			report_json = json.loads(stdout_text)
+		except json.decoder.JSONDecodeError:
+			# Some lvm commands don't return json even though we are asking for it to do so.
+			return process.returncode, stdout_text, stderr_text
+
+		error_msg = get_error_msg(report_json)
+		if error_msg:
+			stderr_text += error_msg
+
+		return process.returncode, report_json, stderr_text
+	else:
+		if cfg.run.value == 0:
+			raise SystemExit
+		# We can bail out before the lvm command finished when we get a signal
+		# which is requesting we exit
+		return -errno.EINTR, "", "operation interrupted"
 
-	return process.returncode, stdout_text, stderr_text
 
 # The actual method which gets called to invoke the lvm command, can vary
 # from forking a new process to using lvm shell
@@ -165,11 +202,11 @@ def _shell_cfg():
 		_t_call = lvm_shell.call_lvm
 		cfg.SHELL_IN_USE = lvm_shell
 		return True
-	except Exception:
+	except Exception as e:
 		_t_call = call_lvm
 		cfg.SHELL_IN_USE = None
-		log_error(traceback.format_exc())
-		log_error("Unable to utilize lvm shell, dropping back to fork & exec")
+		log_error("Unable to utilize lvm shell, dropping "
+				  "back to fork & exec\n%s" % extract_stack_trace(e))
 		return False
 
 
@@ -200,11 +237,15 @@ def time_wrapper(command, debug=False):
 
 	with cmd_lock:
 		start = time.time()
+		meta = LvmExecutionMeta(start, 0, command)
+		# Add the partial metadata to flight recorder, so if the command hangs
+		# we will see what it was.
+		cfg.flightrecorder.add(meta)
 		results = _t_call(command, debug)
 		ended = time.time()
 		total_time += (ended - start)
 		total_count += 1
-		cfg.blackbox.add(LvmExecutionMeta(start, ended, command, *results))
+		meta.completed(ended, *results)
 	return results
 
 
@@ -214,44 +255,11 @@ call = time_wrapper
 # Default cmd
 # Place default arguments for every command here.
 def _dc(cmd, args):
-	c = [cmd, '--noheading', '--separator', '%s' % SEP, '--nosuffix',
-		'--unbuffered', '--units', 'b']
+	c = [cmd, '--nosuffix', '--unbuffered', '--units', 'b']
 	c.extend(args)
 	return c
 
 
-def parse(out):
-	rc = []
-
-	for line in out.split('\n'):
-		# This line includes separators, so process them
-		if SEP in line:
-			elem = line.split(SEP)
-			cleaned_elem = []
-			for e in elem:
-				e = e.strip()
-				cleaned_elem.append(e)
-
-			if len(cleaned_elem) > 1:
-				rc.append(cleaned_elem)
-		else:
-			t = line.strip()
-			if len(t) > 0:
-				rc.append(t)
-	return rc
-
-
-def parse_column_names(out, column_names):
-	lines = parse(out)
-	rc = []
-
-	for i in range(0, len(lines)):
-		d = dict(list(zip(column_names, lines[i])))
-		rc.append(d)
-
-	return rc
-
-
 def options_to_cli_args(options):
 	rc = []
 	for k, v in list(dict(options).items()):
@@ -613,68 +621,20 @@ def lvm_full_report_json():
 		'--configreport', 'vg', '-o', ','.join(vg_columns),
 		'--configreport', 'lv', '-o', ','.join(lv_columns),
 		'--configreport', 'seg', '-o', ','.join(lv_seg_columns),
-		'--configreport', 'pvseg', '-o', ','.join(pv_seg_columns),
-		'--reportformat', 'json'
+		'--configreport', 'pvseg', '-o', ','.join(pv_seg_columns)
 	])
 
+	# We are running the fullreport command, we will ask lvm to output the debug
+	# data, so we can have the required information for lvm to debug the fullreport failures.
+	fn = cfg.lvmdebug.setup()
+	add_config_option(cmd, "--config", "log {level=7 file=%s syslog=0}" % fn)
+
 	rc, out, err = call(cmd)
 	# When we have an exported vg the exit code of lvs or fullreport will be 5
 	if rc == 0 or rc == 5:
-		# With the current implementation, if we are using the shell then we
-		# are using JSON and JSON is returned back to us as it was parsed to
-		# figure out if we completed OK or not
-		if cfg.SHELL_IN_USE:
-			assert(type(out) == dict)
-			return out
-		else:
-			try:
-				return json.loads(out)
-			except json.decoder.JSONDecodeError as joe:
-				log_error("JSONDecodeError %s, \n JSON=\n%s\n" %
-							(str(joe), out))
-				raise joe
-
-	return None
-
-
-def pv_retrieve_with_segs(device=None):
-	d = []
-	err = ""
-	out = ""
-	rc = 0
-
-	columns = ['pv_name', 'pv_uuid', 'pv_fmt', 'pv_size', 'pv_free',
-				'pv_used', 'dev_size', 'pv_mda_size', 'pv_mda_free',
-				'pv_ba_start', 'pv_ba_size', 'pe_start', 'pv_pe_count',
-				'pv_pe_alloc_count', 'pv_attr', 'pv_tags', 'vg_name',
-				'vg_uuid', 'pvseg_start', 'pvseg_size', 'segtype', 'pv_missing']
-
-	# Lvm has some issues where it returns failure when querying pvs when other
-	# operations are in process, see:
-	# https://bugzilla.redhat.com/show_bug.cgi?id=1274085
-	for i in range(0, 10):
-		cmd = _dc('pvs', ['-o', ','.join(columns)])
-
-		if device:
-			cmd.extend(device)
-
-		rc, out, err = call(cmd)
-
-		if rc == 0:
-			d = parse_column_names(out, columns)
-			break
-		else:
-			time.sleep(0.2)
-			log_debug("LVM Bug workaround, retrying pvs command...")
-
-	if rc != 0:
-		msg = "We were unable to get pvs to return without error after " \
-			"trying 10 times, RC=%d, STDERR=(%s), STDOUT=(%s)" % \
-			(rc, err, out)
-		log_error(msg)
-		raise RuntimeError(msg)
-
-	return d
+		assert(type(out) == dict)
+		return out
+	raise LvmBug("'fullreport' exited with code '%d'" % rc)
 
 
 def pv_resize(device, size_bytes, create_options):
@@ -831,53 +791,6 @@ def activate_deactivate(op, name, activate, control_flags, options):
 	return call(cmd)
 
 
-def vg_retrieve(vg_specific):
-	if vg_specific:
-		assert isinstance(vg_specific, list)
-
-	columns = ['vg_name', 'vg_uuid', 'vg_fmt', 'vg_size', 'vg_free',
-				'vg_sysid', 'vg_extent_size', 'vg_extent_count',
-				'vg_free_count', 'vg_profile', 'max_lv', 'max_pv',
-				'pv_count', 'lv_count', 'snap_count', 'vg_seqno',
-				'vg_mda_count', 'vg_mda_free', 'vg_mda_size',
-				'vg_mda_used_count', 'vg_attr', 'vg_tags']
-
-	cmd = _dc('vgs', ['-o', ','.join(columns)])
-
-	if vg_specific:
-		cmd.extend(vg_specific)
-
-	d = []
-	rc, out, err = call(cmd)
-	if rc == 0:
-		d = parse_column_names(out, columns)
-
-	return d
-
-
-def lv_retrieve_with_segments():
-	columns = ['lv_uuid', 'lv_name', 'lv_path', 'lv_size',
-				'vg_name', 'pool_lv_uuid', 'pool_lv', 'origin_uuid',
-				'origin', 'data_percent',
-				'lv_attr', 'lv_tags', 'vg_uuid', 'lv_active', 'data_lv',
-				'metadata_lv', 'seg_pe_ranges', 'segtype', 'lv_parent',
-				'lv_role', 'lv_layout',
-				'snap_percent', 'metadata_percent', 'copy_percent',
-				'sync_percent', 'lv_metadata_size', 'move_pv', 'move_pv_uuid']
-
-	cmd = _dc('lvs', ['-a', '-o', ','.join(columns)])
-	rc, out, err = call(cmd)
-
-	d = []
-
-	if rc == 0:
-		d = parse_column_names(out, columns)
-
-	return d
-
-
 if __name__ == '__main__':
-	pv_data = pv_retrieve_with_segs()
-
-	for p in pv_data:
-		print(str(p))
+	# Leave this for future debug as needed
+	pass

daemons/lvmdbusd/fetch.py

@@ -6,16 +6,16 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
+import errno
 
 from .pv import load_pvs
 from .vg import load_vgs
 from .lv import load_lvs
 from . import cfg
-from .utils import MThreadRunner, log_debug, log_error
+from .utils import MThreadRunner, log_debug, log_error, LvmBug, extract_stack_trace
 import threading
 import queue
 import time
-import traceback
 
 
 def _main_thread_load(refresh=True, emit_signal=True):
@@ -120,81 +120,108 @@ class StateUpdate(object):
 	@staticmethod
 	def update_thread(obj):
 		exception_count = 0
-
 		queued_requests = []
+
+		def set_results(val):
+			nonlocal queued_requests
+			for idx in queued_requests:
+				idx.set_result(val)
+				# Only clear out the requests after we have given them a result
+				# otherwise we can orphan the waiting threads, and they never
+				# wake up if we get an exception
+				queued_requests = []
+
+		def bailing(rv):
+			set_results(rv)
+			try:
+				while True:
+					item = obj.queue.get(False)
+					item.set_result(rv)
+			except queue.Empty:
+				pass
+
+		def _load_args(requests):
+			"""
+			If we have multiple requests in the queue, they might not all have the same options.  If any of the requests
+			have an option set we need to honor it.
+			"""
+			refresh = any([r.refresh for r in requests])
+			emit_signal = any([r.emit_signal for r in requests])
+			cache_refresh = any([r.cache_refresh for r in requests])
+			log = any([r.log for r in requests])
+			need_main_thread = any([r.need_main_thread for r in requests])
+
+			return refresh, emit_signal, cache_refresh, log, need_main_thread
+
+		def _drain_queue(queued, incoming):
+			try:
+				while True:
+					queued.append(incoming.get(block=False))
+			except queue.Empty:
+				pass
+
+		def _handle_error():
+			nonlocal exception_count
+			exception_count += 1
+
+			if exception_count >= 5:
+				log_error("Too many errors in update_thread, exiting daemon")
+				cfg.debug.dump()
+				cfg.flightrecorder.dump()
+				bailing(errno.EFAULT)
+				cfg.exit_daemon()
+			else:
+				# Slow things down when encountering errors
+				cfg.lvmdebug.complete()
+				time.sleep(1)
+
 		while cfg.run.value != 0:
 			# noinspection PyBroadException
 			try:
-				refresh = True
-				emit_signal = True
-				cache_refresh = True
-				log = True
-				need_main_thread = True
-
 				with obj.lock:
 					wait = not obj.deferred
 					obj.deferred = False
 
 				if len(queued_requests) == 0 and wait:
-					queued_requests.append(obj.queue.get(True, 2))
+					# Note: If we don't have anything for 2 seconds we will
+					# get a queue.Empty exception raised here
+					queued_requests.append(obj.queue.get(block=True, timeout=2))
 
 				# Ok we have one or the deferred queue has some,
-				# check if any others
-				try:
-					while True:
-						queued_requests.append(obj.queue.get(False))
-
-				except queue.Empty:
-					pass
+				# check if any others and grab them too
+				_drain_queue(queued_requests, obj.queue)
 
 				if len(queued_requests) > 1:
 					log_debug("Processing %d updates!" % len(queued_requests),
 							'bg_black', 'fg_light_green')
 
-				# We have what we can, run the update with the needed options
-				for i in queued_requests:
-					if not i.refresh:
-						refresh = False
-					if not i.emit_signal:
-						emit_signal = False
-					if not i.cache_refresh:
-						cache_refresh = False
-					if not i.log:
-						log = False
-					if not i.need_main_thread:
-						need_main_thread = False
-
-				num_changes = load(refresh, emit_signal, cache_refresh, log,
-									need_main_thread)
+				num_changes = load(*_load_args(queued_requests))
 				# Update is done, let everyone know!
-				for i in queued_requests:
-					i.set_result(num_changes)
-
-				# Only clear out the requests after we have given them a result
-				# otherwise we can orphan the waiting threads and they never
-				# wake up if we get an exception
-				queued_requests = []
+				set_results(num_changes)
 
 				# We retrieved OK, clear exception count
 				exception_count = 0
 
 			except queue.Empty:
 				pass
+			except SystemExit:
+				break
+			except LvmBug as bug:
+				# If a lvm bug occurred, we will dump the lvm debug data if
+				# we have it.
+				cfg.lvmdebug.dump()
+				log_error(str(bug))
+				_handle_error()
 			except Exception as e:
-				st = traceback.format_exc()
-				log_error("update_thread exception: \n%s" % st)
-				cfg.blackbox.dump()
-				exception_count += 1
-				if exception_count >= 5:
-					for i in queued_requests:
-						i.set_result(e)
+				log_error("update_thread: \n%s" % extract_stack_trace(e))
+				_handle_error()
+			finally:
+				cfg.lvmdebug.complete()
 
-					log_error("Too many errors in update_thread, exiting daemon")
-					cfg.exit_daemon()
-
-				else:
-					# Slow things down when encountering errors
-					time.sleep(1)
+		# Make sure to unblock any that may be waiting before we exit this thread
+		# otherwise they hang forever ...
+		bailing(Exception("update thread exiting"))
+		log_debug("update thread exiting!")
 
 	def __init__(self):
 		self.lock = threading.RLock()

daemons/lvmdbusd/job.py

@@ -226,3 +226,21 @@ class Job(AutomatedProperties):
 	def Uuid(self):
 		import uuid
 		return uuid.uuid1()
+
+	# Override the property "getters" implementation for the job interface, so a user can query a job while the queue
+	# is processing items.  Originally all the property get methods were this way, but we changed this in
+	# e53454d6de07de56736303dd2157c3859f6fa848
+
+	# Properties
+	# noinspection PyUnusedLocal
+	@dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE,
+						 in_signature='ss', out_signature='v')
+	def Get(self, interface_name, property_name):
+		# Note: If we get an exception in this handler we won't know about it,
+		# only the side effect of no returned value!
+		return AutomatedProperties._get_prop(self, interface_name, property_name)
+
+	@dbus.service.method(dbus_interface=dbus.PROPERTIES_IFACE,
+						 in_signature='s', out_signature='a{sv}')
+	def GetAll(self, interface_name):
+		return AutomatedProperties._get_all_prop(self, interface_name)

daemons/lvmdbusd/lv.py

@@ -10,7 +10,7 @@
 from .automatedproperties import AutomatedProperties
 
 from . import utils
-from .utils import vg_obj_path_generate, log_error, _handle_execute
+from .utils import vg_obj_path_generate, log_error, _handle_execute, LvmBug
 import dbus
 from . import cmdhandler
 from . import cfg
@@ -21,11 +21,9 @@ from .utils import n, n32, d
 from .loader import common
 from .state import State
 from . import background
-from .utils import round_size, mt_remove_dbus_objects
+from .utils import round_size, mt_remove_dbus_objects, lvm_column_key
 from .job import JobState
 
-import traceback
-
 
 # Try and build a key for a LV, so that we sort the LVs with least dependencies
 # first.  This may be error prone because of the flexibility LVM
@@ -73,67 +71,74 @@ def lvs_state_retrieve(selection, cache_refresh=True):
 	# don't have information available yet.
 	lvs = sorted(cfg.db.fetch_lvs(selection), key=get_key)
 
-	for l in lvs:
-		if cfg.vdo_support:
-			rc.append(LvStateVdo(
-				l['lv_uuid'], l['lv_name'],
-				l['lv_path'], n(l['lv_size']),
-				l['vg_name'],
-				l['vg_uuid'], l['pool_lv_uuid'],
-				l['pool_lv'], l['origin_uuid'], l['origin'],
-				n32(l['data_percent']), l['lv_attr'],
-				l['lv_tags'], l['lv_active'], l['data_lv'],
-				l['metadata_lv'], l['segtype'], l['lv_role'],
-				l['lv_layout'],
-				n32(l['snap_percent']),
-				n32(l['metadata_percent']),
-				n32(l['copy_percent']),
-				n32(l['sync_percent']),
-				n(l['lv_metadata_size']),
-				l['move_pv'],
-				l['move_pv_uuid'],
-				l['vdo_operating_mode'],
-				l['vdo_compression_state'],
-				l['vdo_index_state'],
-				n(l['vdo_used_size']),
-				d(l['vdo_saving_percent']),
-				l['vdo_compression'],
-				l['vdo_deduplication'],
-				l['vdo_use_metadata_hints'],
-				n32(l['vdo_minimum_io_size']),
-				n(l['vdo_block_map_cache_size']),
-				n32(l['vdo_block_map_era_length']),
-				l['vdo_use_sparse_index'],
-				n(l['vdo_index_memory_size']),
-				n(l['vdo_slab_size']),
-				n32(l['vdo_ack_threads']),
-				n32(l['vdo_bio_threads']),
-				n32(l['vdo_bio_rotation']),
-				n32(l['vdo_cpu_threads']),
-				n32(l['vdo_hash_zone_threads']),
-				n32(l['vdo_logical_threads']),
-				n32(l['vdo_physical_threads']),
-				n32(l['vdo_max_discard']),
-				l['vdo_write_policy'],
-				n32(l['vdo_header_size'])))
-		else:
-			rc.append(LvState(
-				l['lv_uuid'], l['lv_name'],
-				l['lv_path'], n(l['lv_size']),
-				l['vg_name'],
-				l['vg_uuid'], l['pool_lv_uuid'],
-				l['pool_lv'], l['origin_uuid'], l['origin'],
-				n32(l['data_percent']), l['lv_attr'],
-				l['lv_tags'], l['lv_active'], l['data_lv'],
-				l['metadata_lv'], l['segtype'], l['lv_role'],
-				l['lv_layout'],
-				n32(l['snap_percent']),
-				n32(l['metadata_percent']),
-				n32(l['copy_percent']),
-				n32(l['sync_percent']),
-				n(l['lv_metadata_size']),
-				l['move_pv'],
-				l['move_pv_uuid']))
+	try:
+		for l in lvs:
+			if cfg.vdo_support:
+				rc.append(LvStateVdo(
+					l['lv_uuid'], l['lv_name'],
+					l['lv_path'], n(l['lv_size']),
+					l['vg_name'],
+					l['vg_uuid'], l['pool_lv_uuid'],
+					l['pool_lv'], l['origin_uuid'], l['origin'],
+					n32(l['data_percent']), l['lv_attr'],
+					l['lv_tags'], l['lv_active'], l['data_lv'],
+					l['metadata_lv'], l['segtype'], l['lv_role'],
+					l['lv_layout'],
+					n32(l['snap_percent']),
+					n32(l['metadata_percent']),
+					n32(l['copy_percent']),
+					n32(l['sync_percent']),
+					n(l['lv_metadata_size']),
+					l['move_pv'],
+					l['move_pv_uuid'],
+					l['vdo_operating_mode'],
+					l['vdo_compression_state'],
+					l['vdo_index_state'],
+					n(l['vdo_used_size']),
+					d(l['vdo_saving_percent']),
+					l['vdo_compression'],
+					l['vdo_deduplication'],
+					l['vdo_use_metadata_hints'],
+					n32(l['vdo_minimum_io_size']),
+					n(l['vdo_block_map_cache_size']),
+					n32(l['vdo_block_map_era_length']),
+					l['vdo_use_sparse_index'],
+					n(l['vdo_index_memory_size']),
+					n(l['vdo_slab_size']),
+					n32(l['vdo_ack_threads']),
+					n32(l['vdo_bio_threads']),
+					n32(l['vdo_bio_rotation']),
+					n32(l['vdo_cpu_threads']),
+					n32(l['vdo_hash_zone_threads']),
+					n32(l['vdo_logical_threads']),
+					n32(l['vdo_physical_threads']),
+					n32(l['vdo_max_discard']),
+					l['vdo_write_policy'],
+					n32(l['vdo_header_size'])))
+			else:
+				rc.append(LvState(
+					l['lv_uuid'], l['lv_name'],
+					l['lv_path'], n(l['lv_size']),
+					l['vg_name'],
+					l['vg_uuid'], l['pool_lv_uuid'],
+					l['pool_lv'], l['origin_uuid'], l['origin'],
+					n32(l['data_percent']), l['lv_attr'],
+					l['lv_tags'], l['lv_active'], l['data_lv'],
+					l['metadata_lv'], l['segtype'], l['lv_role'],
+					l['lv_layout'],
+					n32(l['snap_percent']),
+					n32(l['metadata_percent']),
+					n32(l['copy_percent']),
+					n32(l['sync_percent']),
+					n(l['lv_metadata_size']),
+					l['move_pv'],
+					l['move_pv_uuid']))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 
@@ -274,15 +279,15 @@ class LvStateVdo(LvState):
 					MetaDataPercent, CopyPercent, SyncPercent,
 					MetaDataSizeBytes, move_pv, move_pv_uuid,
 					vdo_operating_mode, vdo_compression_state, vdo_index_state,
-					vdo_used_size,vdo_saving_percent,vdo_compression,
-					vdo_deduplication,vdo_use_metadata_hints,
-					vdo_minimum_io_size,vdo_block_map_cache_size,
-					vdo_block_map_era_length,vdo_use_sparse_index,
-					vdo_index_memory_size,vdo_slab_size,vdo_ack_threads,
-					vdo_bio_threads,vdo_bio_rotation,vdo_cpu_threads,
-					vdo_hash_zone_threads,vdo_logical_threads,
-					vdo_physical_threads,vdo_max_discard,
-					vdo_write_policy,vdo_header_size):
+					vdo_used_size, vdo_saving_percent, vdo_compression,
+					vdo_deduplication, vdo_use_metadata_hints,
+					vdo_minimum_io_size, vdo_block_map_cache_size,
+					vdo_block_map_era_length, vdo_use_sparse_index,
+					vdo_index_memory_size, vdo_slab_size, vdo_ack_threads,
+					vdo_bio_threads, vdo_bio_rotation, vdo_cpu_threads,
+					vdo_hash_zone_threads, vdo_logical_threads,
+					vdo_physical_threads, vdo_max_discard,
+					vdo_write_policy, vdo_header_size):
 		super(LvStateVdo, self).__init__(Uuid, Name, Path, SizeBytes,
 					vg_name, vg_uuid, pool_lv_uuid, PoolLv,
 					origin_uuid, OriginLv, DataPercent, Attr, Tags, active,
@@ -371,8 +376,8 @@ class LvCommon(AutomatedProperties):
 			return dbus.Struct((self.state.Attr[index],
 				type_map.get(self.state.Attr[index], default)),
 								signature="(ss)")
-		except BaseException:
-			st = traceback.format_exc()
+		except BaseException as b:
+			st = utils.extract_stack_trace(b)
 			log_error("attr_struct: \n%s" % st)
 			return dbus.Struct(('?', 'Unavailable'), signature="(ss)")
 
@@ -595,7 +600,7 @@ class Lv(LvCommon):
 				optional_size = space + 512 - remainder
 
 		LvCommon.handle_execute(*cmdhandler.vg_lv_snapshot(
-			lv_name, snapshot_options,name, optional_size))
+			lv_name, snapshot_options, name, optional_size))
 		full_name = "%s/%s" % (dbo.vg_name_lookup(), name)
 		return cfg.om.get_object_path_by_lvm_id(full_name)
 
@@ -635,7 +640,7 @@ class Lv(LvCommon):
 
 		size_change = new_size_bytes - dbo.SizeBytes
 		LvCommon.handle_execute(*cmdhandler.lv_resize(
-			dbo.lvm_id, size_change,pv_dests, resize_options))
+			dbo.lvm_id, size_change, pv_dests, resize_options))
 		return "/"
 
 	@dbus.service.method(
@@ -744,7 +749,7 @@ class Lv(LvCommon):
 		cfg.worker_q.put(r)
 
 	@staticmethod
-	def _writecache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
+	def _caching_common(method, lv_uuid, lv_name, lv_object_path, cache_options):
 		# Make sure we have a dbus object representing it
 		dbo = LvCommon.validate_dbus_object(lv_uuid, lv_name)
 
@@ -753,7 +758,7 @@ class Lv(LvCommon):
 
 		if lv_to_cache:
 			fcn = lv_to_cache.lv_full_name()
-			rc, out, err = cmdhandler.lv_writecache_lv(
+			rc, out, err = method(
 				dbo.lv_full_name(), fcn, cache_options)
 			if rc == 0:
 				# When we cache an LV, the cache pool and the lv that is getting
@@ -770,9 +775,14 @@ class Lv(LvCommon):
 		else:
 			raise dbus.exceptions.DBusException(
 				LV_INTERFACE, 'LV to cache with object path %s not present!' %
-				lv_object_path)
+							  lv_object_path)
 		return lv_converted
 
+	@staticmethod
+	def _writecache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
+		return Lv._caching_common(cmdhandler.lv_writecache_lv, lv_uuid,
+									lv_name, lv_object_path, cache_options)
+
 	@dbus.service.method(
 		dbus_interface=LV_INTERFACE,
 		in_signature='oia{sv}',
@@ -845,10 +855,10 @@ class LvVdoPool(Lv):
 		cfg.worker_q.put(r)
 
 	@dbus.service.method(
-	dbus_interface=VDO_POOL_INTERFACE,
-	in_signature='ia{sv}',
-	out_signature='o',
-	async_callbacks=('cb', 'cbe'))
+		dbus_interface=VDO_POOL_INTERFACE,
+		in_signature='ia{sv}',
+		out_signature='o',
+		async_callbacks=('cb', 'cbe'))
 	def DisableCompression(self, tmo, comp_options, cb, cbe):
 		r = RequestEntry(
 			tmo, LvVdoPool._enable_disable_compression,
@@ -878,10 +888,10 @@ class LvVdoPool(Lv):
 		cfg.worker_q.put(r)
 
 	@dbus.service.method(
-	dbus_interface=VDO_POOL_INTERFACE,
-	in_signature='ia{sv}',
-	out_signature='o',
-	async_callbacks=('cb', 'cbe'))
+		dbus_interface=VDO_POOL_INTERFACE,
+		in_signature='ia{sv}',
+		out_signature='o',
+		async_callbacks=('cb', 'cbe'))
 	def DisableDeduplication(self, tmo, dedup_options, cb, cbe):
 		r = RequestEntry(
 			tmo, LvVdoPool._enable_disable_deduplication,
@@ -952,33 +962,8 @@ class LvCachePool(Lv):
 
 	@staticmethod
 	def _cache_lv(lv_uuid, lv_name, lv_object_path, cache_options):
-		# Make sure we have a dbus object representing cache pool
-		dbo = LvCommon.validate_dbus_object(lv_uuid, lv_name)
-
-		# Make sure we have dbus object representing lv to cache
-		lv_to_cache = cfg.om.get_object_by_path(lv_object_path)
-
-		if lv_to_cache:
-			fcn = lv_to_cache.lv_full_name()
-			rc, out, err = cmdhandler.lv_cache_lv(
-				dbo.lv_full_name(), fcn, cache_options)
-			if rc == 0:
-				# When we cache an LV, the cache pool and the lv that is getting
-				# cached need to be removed from the object manager and
-				# re-created as their interfaces have changed!
-				mt_remove_dbus_objects((dbo, lv_to_cache))
-				cfg.load()
-
-				lv_converted = cfg.om.get_object_path_by_lvm_id(fcn)
-			else:
-				raise dbus.exceptions.DBusException(
-					LV_INTERFACE,
-					'Exit code %s, stderr = %s' % (str(rc), err))
-		else:
-			raise dbus.exceptions.DBusException(
-				LV_INTERFACE, 'LV to cache with object path %s not present!' %
-				lv_object_path)
-		return lv_converted
+		return Lv._caching_common(cmdhandler.lv_cache_lv, lv_uuid, lv_name,
+									lv_object_path, cache_options)
 
 	@dbus.service.method(
 		dbus_interface=CACHE_POOL_INTERFACE,

daemons/lvmdbusd/lvm_shell_proxy.py.in

@@ -14,12 +14,11 @@
 import subprocess
 import shlex
 import os
-import traceback
+import pty
 import sys
 import tempfile
 import time
 import select
-import copy
 
 try:
 	import simplejson as json
@@ -27,9 +26,9 @@ except ImportError:
 	import json
 
 
-from lvmdbusd.cfg import LVM_CMD
+import lvmdbusd.cfg as cfg
 from lvmdbusd.utils import log_debug, log_error, add_no_notify, make_non_block,\
-							read_decoded
+			read_decoded, extract_stack_trace, LvmBug, get_error_msg
 
 SHELL_PROMPT = "lvm> "
 
@@ -43,10 +42,11 @@ def _quote_arg(arg):
 
 class LVMShellProxy(object):
 
-	# Read until we get prompt back and a result
-	# @param: no_output	Caller expects no output to report FD
-	# Returns stdout, report, stderr (report is JSON!)
-	def _read_until_prompt(self, no_output=False):
+	# Read REPORT FD until we have a complete and valid JSON record or give
+	# up trying to get one.
+	#
+	# Returns stdout, report (JSON), stderr
+	def _read_response(self, no_output=False):
 		stdout = ""
 		report = ""
 		stderr = ""
@@ -58,24 +58,27 @@ class LVMShellProxy(object):
 		# Try reading from all FDs to prevent one from filling up and causing
 		# a hang.  Keep reading until we get the prompt back and the report
 		# FD does not contain valid JSON
-		while keep_reading:
+
+		while keep_reading and cfg.run.value != 0:
 			try:
 				rd_fd = [
-					self.lvm_shell.stdout.fileno(),
+					self.parent_stdout_fd,
 					self.report_stream.fileno(),
-					self.lvm_shell.stderr.fileno()]
+					self.parent_stderr_fd]
 				ready = select.select(rd_fd, [], [], 2)
 
 				for r in ready[0]:
-					if r == self.lvm_shell.stdout.fileno():
-						stdout += read_decoded(self.lvm_shell.stdout)
+					if r == self.parent_stdout_fd:
+						for line in self.parent_stdout.readlines():
+							stdout += line
 					elif r == self.report_stream.fileno():
 						report += read_decoded(self.report_stream)
-					elif r == self.lvm_shell.stderr.fileno():
-						stderr += read_decoded(self.lvm_shell.stderr)
+					elif r == self.parent_stderr_fd:
+						for line in self.parent_stderr.readlines():
+							stderr += line
 
 				# Check to see if the lvm process died on us
-				if self.lvm_shell.poll():
+				if self.lvm_shell.poll() is not None:
 					raise Exception(self.lvm_shell.returncode, "%s" % stderr)
 
 				if stdout.endswith(SHELL_PROMPT):
@@ -99,91 +102,96 @@ class LVMShellProxy(object):
 							extra_passes -= 1
 							if extra_passes <= 0:
 								if len(report):
-									raise ValueError("Invalid json: %s" %
+									raise LvmBug("Invalid json: %s" %
 														report)
 								else:
-									raise ValueError(
+									raise LvmBug(
 										"lvm returned no JSON output!")
 
 			except IOError as ioe:
 				log_debug(str(ioe))
-				pass
+				self.exit_shell()
+				raise ioe
+
+		if keep_reading and cfg.run.value == 0:
+			# We didn't complete as we are shutting down
+			# Try to clean up lvm shell process
+			log_debug("exiting lvm shell as we are shutting down")
+			self.exit_shell()
+			raise SystemExit
 
 		return stdout, report_json, stderr
 
 	def _write_cmd(self, cmd):
-		cmd_bytes = bytes(cmd, "utf-8")
-		num_written = self.lvm_shell.stdin.write(cmd_bytes)
-		assert (num_written == len(cmd_bytes))
-		self.lvm_shell.stdin.flush()
+		self.parent_stdin.write(cmd)
+		self.parent_stdin.flush()
 
 	def __init__(self):
-
 		# Create a temp directory
 		tmp_dir = tempfile.mkdtemp(prefix="lvmdbus_")
 		tmp_file = "%s/lvmdbus_report" % (tmp_dir)
 
-		try:
-			# Lets create fifo for the report output
-			os.mkfifo(tmp_file, 0o600)
-		except FileExistsError:
-			pass
+		# Create a fifo for the report output
+		os.mkfifo(tmp_file, 0o600)
 
-		# We have to open non-blocking as the other side isn't open until
-		# we actually fork the process.
+		# Open the fifo for use to read and for lvm child process to write to.
 		self.report_fd = os.open(tmp_file, os.O_NONBLOCK)
 		self.report_stream = os.fdopen(self.report_fd, 'rb', 0)
+		lvm_fd = os.open(tmp_file, os.O_WRONLY)
 
-		# Setup the environment for using our own socket for reporting
-		local_env = copy.deepcopy(os.environ)
-		local_env["LVM_REPORT_FD"] = "32"
-		local_env["LVM_COMMAND_PROFILE"] = "lvmdbusd"
+		# Set up the environment for using our own socket for reporting and disable the abort
+		# logic if lvm logs too much, which easily happens when utilizing the lvm shell.
+		local_env = {"LC_ALL": "C", "LVM_REPORT_FD": "%s" % lvm_fd, "LVM_COMMAND_PROFILE": "lvmdbusd",
+					 "LVM_LOG_FILE_MAX_LINES": "0"}
 
-		# Disable the abort logic if lvm logs too much, which easily happens
-		# when utilizing the lvm shell.
-		local_env["LVM_LOG_FILE_MAX_LINES"] = "0"
+		# If any env variables contain LVM we will propagate them too
+		for k, v in os.environ.items():
+			if "LVM" in k:
+				local_env[k] = v
+
+		self.parent_stdin_fd, child_stdin_fd = pty.openpty()
+		self.parent_stdout_fd, child_stdout_fd = pty.openpty()
+		self.parent_stderr_fd, child_stderr_fd = pty.openpty()
+		self.parent_stdin = os.fdopen(self.parent_stdin_fd, "w")
+		self.parent_stdout = os.fdopen(self.parent_stdout_fd, "r")
+		self.parent_stderr = os.fdopen(self.parent_stderr_fd, "r")
 
 		# run the lvm shell
 		self.lvm_shell = subprocess.Popen(
-			[LVM_CMD + " 32>%s" % tmp_file],
-			stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=local_env,
-			stderr=subprocess.PIPE, close_fds=True, shell=True)
+			[cfg.LVM_CMD],
+			stdin=child_stdin_fd,
+			stdout=child_stdout_fd, env=local_env,
+			stderr=child_stderr_fd, close_fds=True,
+			pass_fds=(lvm_fd,), shell=False)
 
 		try:
-			make_non_block(self.lvm_shell.stdout)
-			make_non_block(self.lvm_shell.stderr)
+			make_non_block(self.parent_stdout_fd)
+			make_non_block(self.parent_stderr_fd)
+
+			# Close our copies of the child FDs there were created with the fork, we don't need them open.
+			os.close(lvm_fd)
+			os.close(child_stdin_fd)
+			os.close(child_stdout_fd)
+			os.close(child_stderr_fd)
 
 			# wait for the first prompt
-			errors = self._read_until_prompt(no_output=True)[2]
+			log_debug("waiting for first prompt...")
+			errors = self._read_response(no_output=True)[2]
 			if errors and len(errors):
-				raise RuntimeError(errors)
+				raise LvmBug(errors)
+			log_debug("lvm prompt read!!!")
 		except:
 			raise
 		finally:
-			# These will get deleted when the FD count goes to zero so we
+			# These will get deleted when the FD count goes to zero, so we
 			# can be sure to clean up correctly no matter how we finish
 			os.unlink(tmp_file)
 			os.rmdir(tmp_dir)
 
-	def get_error_msg(self):
-		# We got an error, lets go fetch the error message
+	def get_last_log(self):
 		self._write_cmd('lastlog\n')
-
-		# read everything from the STDOUT to the next prompt
-		stdout, report_json, stderr = self._read_until_prompt()
-		if 'log' in report_json:
-			error_msg = ""
-			# Walk the entire log array and build an error string
-			for log_entry in report_json['log']:
-				if log_entry['log_type'] == "error":
-					if error_msg:
-						error_msg += ', ' + log_entry['log_message']
-					else:
-						error_msg = log_entry['log_message']
-
-			return error_msg
-
-		return 'No error reason provided! (missing "log" section)'
+		report_json = self._read_response()[1]
+		return get_error_msg(report_json)
 
 	def call_lvm(self, argv, debug=False):
 		rc = 1
@@ -204,21 +212,29 @@ class LVMShellProxy(object):
 		self._write_cmd(cmd)
 
 		# read everything from the STDOUT to the next prompt
-		stdout, report_json, stderr = self._read_until_prompt()
+		stdout, report_json, stderr = self._read_response()
 
 		# Parse the report to see what happened
 		if 'log' in report_json:
 			ret_code = int(report_json['log'][-1:][0]['log_ret_code'])
 			# If we have an exported vg we get a log_ret_code == 5 when
 			# we do a 'fullreport'
+			# Note: 0 == error
 			if (ret_code == 1) or (ret_code == 5 and argv[0] == 'fullreport'):
 				rc = 0
 			else:
-				error_msg = self.get_error_msg()
+				# Depending on where lvm fails the command, it may not have anything
+				# to report for "lastlog", so we need to check for a message in the
+				# report json too.
+				error_msg = self.get_last_log()
+				if error_msg is None:
+					error_msg = get_error_msg(report_json)
+					if error_msg is None:
+						error_msg = 'No error reason provided! (missing "log" section)'
 
 		if debug or rc != 0:
-			log_error(('CMD: %s' % cmd))
-			log_error(("EC = %d" % rc))
+			log_error(("CMD= %s" % cmd))
+			log_error(("EC= %d" % rc))
 			log_error(("ERROR_MSG=\n %s\n" % error_msg))
 
 		return rc, report_json, error_msg
@@ -226,35 +242,58 @@ class LVMShellProxy(object):
 	def exit_shell(self):
 		try:
 			self._write_cmd('exit\n')
-		except Exception as e:
-			log_error(str(e))
+			self.lvm_shell.wait(1)
+			self.lvm_shell = None
+		except Exception as _e:
+			log_error(str(_e))
 
 	def __del__(self):
-		try:
-			self.lvm_shell.terminate()
-		except:
-			pass
+		# Note: When we are shutting down the daemon and the main process has already exited
+		# and this gets called we have a limited set of things we can do, like we cannot call
+		# log_error as _common_log is None!!!
+		if self.lvm_shell is not None:
+			try:
+				self.lvm_shell.wait(1)
+			except subprocess.TimeoutExpired:
+				print("lvm shell child process did not exit as instructed, sending SIGTERM")
+				cfg.ignore_sigterm = True
+				self.lvm_shell.terminate()
+				child_exit_code = self.lvm_shell.wait(1)
+				print("lvm shell process exited with %d" % child_exit_code)
 
 
 if __name__ == "__main__":
-	shell = LVMShellProxy()
-	in_line = "start"
+	print("USING LVM BINARY: %s " % cfg.LVM_CMD)
+
 	try:
-		while in_line:
-			in_line = input("lvm> ")
-			if in_line:
-				start = time.time()
-				ret, out, err = shell.call_lvm(in_line.split())
-				end = time.time()
+		if len(sys.argv) > 1 and sys.argv[1] == "bisect":
+			shell = LVMShellProxy()
+			shell.exit_shell()
+		else:
+			shell = LVMShellProxy()
+			in_line = "start"
+			try:
+				while in_line:
+					in_line = input("lvm> ")
+					if in_line:
+						if in_line == "exit":
+							shell.exit_shell()
+							sys.exit(0)
+						start = time.time()
+						ret, out, err = shell.call_lvm(in_line.split())
+						end = time.time()
 
-				print(("RC: %d" % ret))
-				print(("OUT:\n%s" % out))
-				print(("ERR:\n%s" % err))
+						print(("RC: %d" % ret))
+						print(("OUT:\n%s" % out))
+						print(("ERR:\n%s" % err))
 
-				print("Command     = %f seconds" % (end - start))
-	except KeyboardInterrupt:
-		pass
-	except EOFError:
-		pass
-	except Exception:
-		traceback.print_exc(file=sys.stdout)
+						print("Command     = %f seconds" % (end - start))
+			except KeyboardInterrupt:
+				pass
+			except EOFError:
+				pass
+	except Exception as e:
+		log_error("main process exiting on exception!\n%s" % extract_stack_trace(e))
+		sys.exit(1)
+
+	sys.exit(0)

daemons/lvmdbusd/lvmdb.py.in

@@ -13,14 +13,13 @@ from collections import OrderedDict
 
 import pprint as prettyprint
 import os
-import sys
 
 from lvmdbusd import cmdhandler
-from lvmdbusd.utils import log_debug, log_error
+from lvmdbusd.utils import log_debug, log_error, lvm_column_key, LvmBug
 
 
 class DataStore(object):
-	def __init__(self, usejson=True, vdo_support=False):
+	def __init__(self, vdo_support=False):
 		self.pvs = {}
 		self.vgs = {}
 		self.lvs = {}
@@ -35,41 +34,9 @@ class DataStore(object):
 		self.lvs_in_vgs = {}
 		self.pvs_in_vgs = {}
 
-		# self.refresh()
 		self.num_refreshes = 0
-
-		if usejson:
-			self.json = cmdhandler.supports_json()
-		else:
-			self.json = usejson
-
 		self.vdo_support = vdo_support
 
-	@staticmethod
-	def _insert_record(table, key, record, allowed_multiple):
-		if key in table:
-			existing = table[key]
-
-			for rec_k, rec_v in record.items():
-				if rec_k in allowed_multiple:
-					# This column name allows us to store multiple value for
-					# each type
-					if not isinstance(existing[rec_k], list):
-						existing_value = existing[rec_k]
-						existing[rec_k] = [existing_value, rec_v]
-					else:
-						existing[rec_k].append(rec_v)
-				else:
-					# If something is not expected to have changing values
-					# lets ensure that
-					if existing[rec_k] != rec_v:
-						raise RuntimeError(
-							"existing[%s]=%s != %s" %
-							(rec_k, str(existing[rec_k]),
-							str(rec_v)))
-		else:
-			table[key] = record
-
 	@staticmethod
 	def _pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup):
 		for p in c_pvs.values():
@@ -84,22 +51,6 @@ class DataStore(object):
 			# Lookup for translating between /dev/<name> and pv uuid
 			c_lookup[p['pv_name']] = p['pv_uuid']
 
-	@staticmethod
-	def _parse_pvs(_pvs):
-		pvs = sorted(_pvs, key=lambda pk: pk['pv_name'])
-
-		c_pvs = OrderedDict()
-		c_lookup = {}
-		c_pvs_in_vgs = {}
-
-		for p in pvs:
-			DataStore._insert_record(
-				c_pvs, p['pv_uuid'], p,
-				['pvseg_start', 'pvseg_size', 'segtype'])
-
-		DataStore._pvs_parse_common(c_pvs, c_pvs_in_vgs, c_lookup)
-		return c_pvs, c_lookup, c_pvs_in_vgs
-
 	@staticmethod
 	def _parse_pvs_json(_all):
 
@@ -107,7 +58,7 @@ class DataStore(object):
 		c_lookup = {}
 		c_pvs_in_vgs = {}
 
-		# Each item item in the report is a collection of information pertaining
+		# Each item in the report is a collection of information pertaining
 		# to the vg
 		for r in _all['report']:
 			tmp_pv = []
@@ -141,28 +92,6 @@ class DataStore(object):
 
 		return c_pvs, c_lookup, c_pvs_in_vgs
 
-	@staticmethod
-	def _parse_vgs(_vgs):
-		vgs = sorted(_vgs, key=lambda vk: vk['vg_uuid'])
-
-		c_vgs = OrderedDict()
-		c_lookup = {}
-
-		for i in vgs:
-			vg_name = i['vg_name']
-
-			# Lvm allows duplicate vg names.  When this occurs, each subsequent
-			# matching VG name will be called vg_name:vg_uuid.  Note: ':' is an
-			# invalid character for lvm VG names
-			if vg_name in c_lookup:
-				vg_name = "%s:%s" % (vg_name, i['vg_uuid'])
-				i['vg_name'] = vg_name
-
-			c_lookup[vg_name] = i['vg_uuid']
-			DataStore._insert_record(c_vgs, i['vg_uuid'], i, [])
-
-		return c_vgs, c_lookup
-
 	@staticmethod
 	def _parse_vgs_json(_all):
 
@@ -227,28 +156,12 @@ class DataStore(object):
 
 		return c_lvs, c_lvs_in_vgs, c_lvs_hidden, c_lv_full_lookup
 
-	@staticmethod
-	def _parse_lvs(_lvs):
-		lvs = sorted(_lvs, key=lambda vk: vk['lv_name'])
-
-		c_lvs = OrderedDict()
-		c_lv_full_lookup = OrderedDict()
-
-		for i in lvs:
-			full_name = "%s/%s" % (i['vg_name'], i['lv_name'])
-			c_lv_full_lookup[full_name] = i['lv_uuid']
-			DataStore._insert_record(
-				c_lvs, i['lv_uuid'], i,
-				['seg_pe_ranges', 'segtype'])
-
-		return DataStore._parse_lvs_common(c_lvs, c_lv_full_lookup)
-
 	def _parse_lvs_json(self, _all):
 
 		c_lvs = OrderedDict()
 		c_lv_full_lookup = {}
 
-		# Each item item in the report is a collection of information pertaining
+		# Each item in the report is a collection of information pertaining
 		# to the vg
 		for r in _all['report']:
 			# Get the lv data for this VG.
@@ -396,12 +309,12 @@ class DataStore(object):
 		:param log  Add debug log entry/exit messages
 		:return: None
 		"""
-		self.num_refreshes += 1
-		if log:
-			log_debug("lvmdb - refresh entry")
+		try:
+			self.num_refreshes += 1
+			if log:
+				log_debug("lvmdb - refresh entry")
 
-		# Grab everything first then parse it
-		if self.json:
+			# Grab everything first then parse it
 			# Do a single lvm retrieve for everything in json
 			a = cmdhandler.lvm_full_report_json()
 
@@ -409,29 +322,25 @@ class DataStore(object):
 			_vgs, _vgs_lookup = self._parse_vgs_json(a)
 			_lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs_json(a)
 
-		else:
-			_raw_pvs = cmdhandler.pv_retrieve_with_segs()
-			_raw_vgs = cmdhandler.vg_retrieve(None)
-			_raw_lvs = cmdhandler.lv_retrieve_with_segments()
+			# Set all
+			self.pvs = _pvs
+			self.pv_path_to_uuid = _pvs_lookup
+			self.vg_name_to_uuid = _vgs_lookup
+			self.lv_full_name_to_uuid = _lvs_lookup
 
-			_pvs, _pvs_lookup, _pvs_in_vgs = self._parse_pvs(_raw_pvs)
-			_vgs, _vgs_lookup = self._parse_vgs(_raw_vgs)
-			_lvs, _lvs_in_vgs, _lvs_hidden, _lvs_lookup = self._parse_lvs(_raw_lvs)
+			self.vgs = _vgs
+			self.lvs = _lvs
+			self.lvs_in_vgs = _lvs_in_vgs
+			self.pvs_in_vgs = _pvs_in_vgs
+			self.lvs_hidden = _lvs_hidden
 
-		# Set all
-		self.pvs = _pvs
-		self.pv_path_to_uuid = _pvs_lookup
-		self.vg_name_to_uuid = _vgs_lookup
-		self.lv_full_name_to_uuid = _lvs_lookup
-
-		self.vgs = _vgs
-		self.lvs = _lvs
-		self.lvs_in_vgs = _lvs_in_vgs
-		self.pvs_in_vgs = _pvs_in_vgs
-		self.lvs_hidden = _lvs_hidden
-
-		# Create lookup table for which LV and segments are on each PV
-		self.pv_lvs, self.lv_pvs = self._parse_pv_in_lvs()
+			# Create lookup table for which LV and segments are on each PV
+			self.pv_lvs, self.lv_pvs = self._parse_pv_in_lvs()
+		except KeyError as ke:
+			key = ke.args[0]
+			if lvm_column_key(key):
+				raise LvmBug("missing JSON key: '%s'" % key)
+			raise ke
 
 		if log:
 			log_debug("lvmdb - refresh exit")
@@ -451,10 +360,13 @@ class DataStore(object):
 			return rc
 
 	def pv_missing(self, pv_uuid):
+		# The uuid might not be a PV, default to false
 		if pv_uuid in self.pvs:
 			if self.pvs[pv_uuid]['pv_missing'] == '':
 				return False
-		return True
+			else:
+				return True
+		return False
 
 	def fetch_vgs(self, vg_name):
 		if not vg_name:
@@ -527,13 +439,7 @@ class DataStore(object):
 if __name__ == "__main__":
 	pp = prettyprint.PrettyPrinter(indent=4)
 
-	use_json = False
-
-	if len(sys.argv) != 1:
-		print(len(sys.argv))
-		use_json = True
-
-	ds = DataStore(use_json)
+	ds = DataStore()
 	ds.refresh()
 
 	print("PVS")

daemons/lvmdbusd/main.py

@@ -22,14 +22,13 @@ from . import lvmdb
 from gi.repository import GLib
 from .fetch import StateUpdate
 from .manager import Manager
-import traceback
 import queue
 from . import udevwatch
-from .utils import log_debug, log_error
+from .utils import log_debug, log_error, log_msg, DebugMessages
 import argparse
 import os
 import sys
-from .cmdhandler import LvmFlightRecorder, supports_vdo
+from .cmdhandler import LvmFlightRecorder, supports_vdo, supports_json
 from .request import RequestEntry
 
 
@@ -50,12 +49,15 @@ def process_request():
 			log_debug("Method complete: %s" % str(req.method))
 		except queue.Empty:
 			pass
-		except Exception:
-			st = traceback.format_exc()
+		except SystemExit:
+			break
+		except Exception as e:
+			st = utils.extract_stack_trace(e)
 			utils.log_error("process_request exception: \n%s" % st)
+	log_debug("process_request thread exiting!")
 
 
-def check_bb_size(value):
+def check_fr_size(value):
 	v = int(value)
 	if v < 0:
 		raise argparse.ArgumentTypeError(
@@ -77,13 +79,13 @@ def install_signal_handlers():
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGHUP, utils.handler, signal.SIGHUP)
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGINT, utils.handler, signal.SIGINT)
 		signal_add(GLib.PRIORITY_HIGH, signal.SIGUSR1, utils.handler, signal.SIGUSR1)
+		signal_add(GLib.PRIORITY_HIGH, signal.SIGUSR2, utils.handler, signal.SIGUSR2)
+		signal_add(GLib.PRIORITY_HIGH, signal.SIGTERM, utils.handler, signal.SIGTERM)
 	else:
 		log_error("GLib.unix_signal_[add|add_full] are NOT available!")
 
 
-def main():
-	start = time.time()
-	# Add simple command line handling
+def process_args():
 	parser = argparse.ArgumentParser()
 	parser.add_argument(
 		"--udev", action='store_true',
@@ -104,101 +106,141 @@ def main():
 		default=False,
 		dest='use_lvm_shell')
 	parser.add_argument(
-		"--blackboxsize",
-		help="Size of the black box flight recorder, 0 to disable",
+		"--frsize",
+		help="Size of the flight recorder (num. entries), 0 to disable (signal 12 to dump)",
 		default=10,
-		type=check_bb_size,
-		dest='bb_size')
+		type=check_fr_size,
+		dest='fr_size')
 
-	use_session = os.getenv('LVMDBUSD_USE_SESSION', False)
+	args = parser.parse_args()
 
-	# Ensure that we get consistent output for parsing stdout/stderr
+	if not args.use_json:
+		log_error("Daemon no longer supports lvm without JSON support, exiting!")
+		sys.exit(1)
+	else:
+		if not supports_json():
+			log_error("Un-supported version of LVM, daemon requires JSON output, exiting!")
+			sys.exit(1)
+
+	# Add udev watching
+	if args.use_udev:
+		# Make sure this msg ends up in the journal, so we know
+		log_msg('The --udev option is no longer supported,'
+				'the daemon always uses a combination of dbus notify from lvm tools and udev')
+
+	return args
+
+
+def running_under_systemd():
+	""""
+	Checks to see if we are running under systemd, by checking damon fd 0, 1
+	systemd sets stdin to /dev/null and 1 & 2 are a socket
+	"""
+	base = "/proc/self/fd"
+	stdout = os.readlink("%s/0" % base)
+	if stdout == "/dev/null":
+		stdout = os.readlink("%s/1" % base)
+		if "socket" in stdout:
+			return True
+	return False
+
+
+def main():
+	start = time.time()
+	use_session = os.getenv('LVM_DBUSD_USE_SESSION', False)
+
+	# Ensure that we get consistent output for parsing stdout/stderr and that we
+	# are using the lvmdbusd profile.
 	os.environ["LC_ALL"] = "C"
+	os.environ["LVM_COMMAND_PROFILE"] = "lvmdbusd"
+
+	# Save off the debug data needed for lvm team to debug issues
+	# only used for 'fullreport' at this time.
+	cfg.lvmdebug = utils.LvmDebugData()
+
+	# Indicator if we are running under systemd
+	cfg.systemd = running_under_systemd()
+
+	# Add simple command line handling
+	cfg.args = process_args()
 
-	cfg.args = parser.parse_args()
 	cfg.create_request_entry = RequestEntry
 
 	# We create a flight recorder in cmdhandler too, but we replace it here
 	# as the user may be specifying a different size.  The default one in
 	# cmdhandler is for when we are running other code with a different main.
-	cfg.blackbox = LvmFlightRecorder(cfg.args.bb_size)
+	cfg.flightrecorder = LvmFlightRecorder(cfg.args.fr_size)
 
-	if cfg.args.use_lvm_shell and not cfg.args.use_json:
-		log_error("You cannot specify --lvmshell and --nojson")
-		sys.exit(1)
+	# Create a circular buffer for debug logs
+	cfg.debug = DebugMessages()
+
+	log_debug("Using lvm binary: %s" % cfg.LVM_CMD)
 
 	# We will dynamically add interfaces which support vdo if it
 	# exists.
 	cfg.vdo_support = supports_vdo()
 
-	if cfg.vdo_support and not cfg.args.use_json:
-		log_error("You cannot specify --nojson when lvm has VDO support")
-		sys.exit(1)
-
 	# List of threads that we start up
 	thread_list = []
 
 	install_signal_handlers()
 
-	dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
-	dbus.mainloop.glib.threads_init()
+	with utils.LockFile(cfg.LOCK_FILE):
+		dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+		dbus.mainloop.glib.threads_init()
 
-	cmdhandler.set_execution(cfg.args.use_lvm_shell)
+		cmdhandler.set_execution(cfg.args.use_lvm_shell)
 
-	if use_session:
-		cfg.bus = dbus.SessionBus()
-	else:
-		cfg.bus = dbus.SystemBus()
-	# The base name variable needs to exist for things to work.
-	# noinspection PyUnusedLocal
-	base_name = dbus.service.BusName(BUS_NAME, cfg.bus)
-	cfg.om = Lvm(BASE_OBJ_PATH)
-	cfg.om.register_object(Manager(MANAGER_OBJ_PATH))
+		if use_session:
+			cfg.bus = dbus.SessionBus()
+		else:
+			cfg.bus = dbus.SystemBus()
+		# The base name variable needs to exist for things to work.
+		# noinspection PyUnusedLocal
+		base_name = dbus.service.BusName(BUS_NAME, cfg.bus)
+		cfg.om = Lvm(BASE_OBJ_PATH)
+		cfg.om.register_object(Manager(MANAGER_OBJ_PATH))
 
-	cfg.db = lvmdb.DataStore(cfg.args.use_json, cfg.vdo_support)
+		cfg.db = lvmdb.DataStore(vdo_support=cfg.vdo_support)
 
-	# Using a thread to process requests, we cannot hang the dbus library
-	# thread that is handling the dbus interface
-	thread_list.append(
-		threading.Thread(target=process_request, name='process_request'))
+		# Using a thread to process requests, we cannot hang the dbus library
+		# thread that is handling the dbus interface
+		thread_list.append(
+			threading.Thread(target=process_request, name='process_request'))
 
-	# Have a single thread handling updating lvm and the dbus model so we
-	# don't have multiple threads doing this as the same time
-	updater = StateUpdate()
-	thread_list.append(updater.thread)
+		# Have a single thread handling updating lvm and the dbus model so we
+		# don't have multiple threads doing this as the same time
+		updater = StateUpdate()
+		thread_list.append(updater.thread)
 
-	cfg.load = updater.load
+		cfg.load = updater.load
 
-	cfg.loop = GLib.MainLoop()
+		cfg.loop = GLib.MainLoop()
 
-	for thread in thread_list:
-		thread.damon = True
-		thread.start()
+		for thread in thread_list:
+			thread.damon = True
+			thread.start()
 
-	# Add udev watching
-	if cfg.args.use_udev:
-		log_debug('Utilizing udev to trigger updates')
+		# In all cases we are going to monitor for udev until we get an
+		# ExternalEvent.  In the case where we get an external event and the user
+		# didn't specify --udev we will stop monitoring udev
+		udevwatch.add()
 
-	# In all cases we are going to monitor for udev until we get an
-	# ExternalEvent.  In the case where we get an external event and the user
-	# didn't specify --udev we will stop monitoring udev
-	udevwatch.add()
+		end = time.time()
+		log_debug(
+			'Service ready! total time= %.4f, lvm time= %.4f count= %d' %
+			(end - start, cmdhandler.total_time, cmdhandler.total_count),
+			'bg_black', 'fg_light_green')
 
-	end = time.time()
-	log_debug(
-		'Service ready! total time= %.4f, lvm time= %.4f count= %d' %
-		(end - start, cmdhandler.total_time, cmdhandler.total_count),
-		'bg_black', 'fg_light_green')
+		try:
+			if cfg.run.value != 0:
+				cfg.loop.run()
+				udevwatch.remove()
 
-	try:
-		if cfg.run.value != 0:
-			cfg.loop.run()
-			udevwatch.remove()
-
-			for thread in thread_list:
-				thread.join()
-	except KeyboardInterrupt:
-		# If we are unable to register signal handler, we will end up here when
-		# the service gets a ^C or a kill -2 <parent pid>
-		utils.handler(signal.SIGINT)
+				for thread in thread_list:
+					thread.join()
+		except KeyboardInterrupt:
+			# If we are unable to register signal handler, we will end up here when
+			# the service gets a ^C or a kill -2 <parent pid>
+			utils.handler(signal.SIGINT)
 	return 0

daemons/lvmdbusd/manager.py

@@ -137,7 +137,8 @@ class Manager(AutomatedProperties):
 		"""
 		Dump the flight recorder to syslog
 		"""
-		cfg.blackbox.dump()
+		cfg.debug.dump()
+		cfg.flightrecorder.dump()
 
 	@staticmethod
 	def _lookup_by_lvm_id(key):
@@ -194,6 +195,7 @@ class Manager(AutomatedProperties):
 	def _external_event(command):
 		utils.log_debug("Processing _external_event= %s" % command,
 							'bg_black', 'fg_orange')
+		cfg.got_external_event = True
 		cfg.load()
 
 	@dbus.service.method(
@@ -201,14 +203,6 @@ class Manager(AutomatedProperties):
 		in_signature='s', out_signature='i')
 	def ExternalEvent(self, command):
 		utils.log_debug("ExternalEvent %s" % command)
-		# If a user didn't explicitly specify udev, we will turn it off now.
-		if not cfg.args.use_udev:
-			if udevwatch.remove():
-				utils.log_debug("ExternalEvent received, disabling "
-								"udev monitoring")
-				# We are dependent on external events now to stay current!
-				cfg.got_external_event = True
-
 		r = RequestEntry(
 			-1, Manager._external_event, (command,), None, None, False)
 		cfg.worker_q.put(r)

daemons/lvmdbusd/objectmanager.py

@@ -9,12 +9,11 @@
 
 import sys
 import threading
-import traceback
 import dbus
 import os
 import copy
 from . import cfg
-from .utils import log_debug, pv_obj_path_generate, log_error
+from .utils import log_debug, pv_obj_path_generate, log_error, extract_stack_trace
 from .automatedproperties import AutomatedProperties
 
 
@@ -40,8 +39,8 @@ class ObjectManager(AutomatedProperties):
 				for k, v in list(obj._objects.items()):
 					path, props = v[0].emit_data()
 					rc[path] = props
-			except Exception:
-				traceback.print_exc(file=sys.stdout)
+			except Exception as e:
+				log_error("_get_managed_objects exception, bailing: \n%s" % extract_stack_trace(e))
 				sys.exit(1)
 			return rc
 
@@ -53,15 +52,6 @@ class ObjectManager(AutomatedProperties):
 									(self, ), cb, cbe, False)
 		cfg.worker_q.put(r)
 
-	def locked(self):
-		"""
-		If some external code need to run across a number of different
-		calls into ObjectManager while blocking others they can use this method
-		to lock others out.
-		:return:
-		"""
-		return ObjectManagerLock(self.rlock)
-
 	@dbus.service.signal(
 		dbus_interface="org.freedesktop.DBus.ObjectManager",
 		signature='oa{sa{sv}}')
@@ -287,7 +277,7 @@ class ObjectManager(AutomatedProperties):
 		register it with the object manager for the specified uuid & lvm_id.
 		Note: If path create is not None, uuid and lvm_id cannot be equal
 		:param uuid: The uuid for the lvm object we are searching for
-		:param lvm_id: The lvm name (eg. pv device path, vg name, lv full name)
+		:param lvm_id: The lvm name (e.g. pv device path, vg name, lv full name)
 		:param path_create: If not None, create the path using this function if
 				we fail to find the object by uuid or lvm_id.
 		:returns None if lvm asset not found and path_create == None otherwise
@@ -308,12 +298,11 @@ class ObjectManager(AutomatedProperties):
 				# We have a uuid and a lvm_id we can do sanity checks to ensure
 				# that they are consistent
 
-				# If a PV is missing it's device path is '[unknown]' or some
+				# If a PV is missing its device path is '[unknown]' or some
 				# other text derivation of unknown.  When we find that a PV is
-				# missing we will clear out the lvm_id as it's likely not unique
-				# and thus not useful and potentially harmful for lookups.
-				if path_create == pv_obj_path_generate and \
-						cfg.db.pv_missing(uuid):
+				# missing we will clear out the lvm_id as it's not unique
+				# and thus not useful and harmful for lookups.
+				if cfg.db.pv_missing(uuid):
 					lvm_id = None
 
 				# Lets check for the uuid first
@@ -337,29 +326,3 @@ class ObjectManager(AutomatedProperties):
 			#	(uuid, lvm_id, str(path_create), path))
 
 			return path
-
-
-class ObjectManagerLock(object):
-	"""
-	The sole purpose of this class is to allow other code the ability to
-	lock the object manager using a `with` statement, eg.
-
-	with cfg.om.locked():
-		# Do stuff with object manager
-
-	This will ensure that the lock is always released (assuming this is done
-	correctly)
-	"""
-
-	def __init__(self, recursive_lock):
-		self._lock = recursive_lock
-
-	def __enter__(self):
-		# Acquire lock
-		self._lock.acquire()
-
-	# noinspection PyUnusedLocal
-	def __exit__(self, e_type, e_value, e_traceback):
-		# Release lock
-		self._lock.release()
-		self._lock = None

daemons/lvmdbusd/pv.py

@@ -14,11 +14,11 @@ import dbus
 from .cfg import PV_INTERFACE
 from . import cmdhandler
 from .utils import vg_obj_path_generate, n, pv_obj_path_generate, \
-	lv_object_path_method, _handle_execute
+	lv_object_path_method, _handle_execute, lvm_column_key
 from .loader import common
 from .request import RequestEntry
 from .state import State
-from .utils import round_size
+from .utils import round_size, LvmBug
 
 
 # noinspection PyUnusedLocal
@@ -28,16 +28,23 @@ def pvs_state_retrieve(selection, cache_refresh=True):
 	if cache_refresh:
 		cfg.db.refresh()
 
-	for p in cfg.db.fetch_pvs(selection):
-		rc.append(
-			PvState(
-				p["pv_name"], p["pv_uuid"], p["pv_name"],
-				p["pv_fmt"], n(p["pv_size"]), n(p["pv_free"]),
-				n(p["pv_used"]), n(p["dev_size"]), n(p["pv_mda_size"]),
-				n(p["pv_mda_free"]), int(p["pv_ba_start"]),
-				n(p["pv_ba_size"]), n(p["pe_start"]),
-				int(p["pv_pe_count"]), int(p["pv_pe_alloc_count"]),
-				p["pv_attr"], p["pv_tags"], p["vg_name"], p["vg_uuid"]))
+	try:
+		for p in cfg.db.fetch_pvs(selection):
+			rc.append(
+				PvState(
+					p["pv_name"], p["pv_uuid"], p["pv_name"],
+					p["pv_fmt"], n(p["pv_size"]), n(p["pv_free"]),
+					n(p["pv_used"]), n(p["dev_size"]), n(p["pv_mda_size"]),
+					n(p["pv_mda_free"]), int(p["pv_ba_start"]),
+					n(p["pv_ba_size"]), n(p["pe_start"]),
+					int(p["pv_pe_count"]), int(p["pv_pe_alloc_count"]),
+					p["pv_attr"], p["pv_tags"], p["vg_name"], p["vg_uuid"]))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 

daemons/lvmdbusd/request.py

@@ -7,13 +7,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+import dbus
 import threading
 # noinspection PyUnresolvedReferences
 from gi.repository import GLib
 from .job import Job
 from . import cfg
-import traceback
-from .utils import log_error, mt_async_call
+from .utils import log_error, mt_async_call, extract_stack_trace
 
 
 class RequestEntry(object):
@@ -71,13 +71,23 @@ class RequestEntry(object):
 		try:
 			result = self.method(*self.arguments)
 			self.register_result(result)
+		except SystemExit as se:
+			self.register_error(-1, str(se), se)
+			raise se
+		except dbus.exceptions.DBusException as dbe:
+			# This is an expected error path when something goes awry that
+			# we handled
+			self.register_error(-1, str(dbe), dbe)
 		except Exception as e:
 			# Use the request entry to return the result as the client may
 			# have gotten a job by the time we hit an error
-			# Lets get the stacktrace and set that to the error message
-			st = traceback.format_exc()
-			cfg.blackbox.dump()
-			log_error("Exception returned to client: \n%s" % st)
+			# Lets set the exception text as the error message and log the
+			# exception in the journal for figuring out what went wrong.
+			cfg.debug.dump()
+			cfg.flightrecorder.dump()
+			tb = extract_stack_trace(e)
+			log_error("While processing %s: we encountered\n%s" % (str(self.method), tb))
+			log_error("Error returned to client: %s" % str(e))
 			self.register_error(-1, str(e), e)
 
 	def is_done(self):

daemons/lvmdbusd/udevwatch.py

@@ -52,20 +52,30 @@ def filter_event(action, device):
 	# when appropriate.
 	refresh = False
 
+	# Ignore everything but change
+	if action != 'change':
+		return
+
 	if 'ID_FS_TYPE' in device:
 		fs_type_new = device['ID_FS_TYPE']
-
 		if 'LVM' in fs_type_new:
-			refresh = True
+			# If we get a lvm related udev event for a block device
+			# we don't know about, it's either a pvcreate which we
+			# would handle with the dbus notification or something
+			# copied a pv signature onto a block device, this is
+			# required to catch the latter.
+			if not cfg.om.get_object_by_lvm_id(device['DEVNAME']):
+				refresh = True
 		elif fs_type_new == '':
 			# Check to see if the device was one we knew about
 			if 'DEVNAME' in device:
-				found = cfg.om.get_object_by_lvm_id(device['DEVNAME'])
-				if found:
+				if cfg.om.get_object_by_lvm_id(device['DEVNAME']):
 					refresh = True
-
-	if 'DM_LV_NAME' in device:
-		refresh = True
+	else:
+		# This handles the wipefs -a path
+		if not refresh and 'DEVNAME' in device:
+			if cfg.om.get_object_by_lvm_id(device['DEVNAME']):
+				refresh = True
 
 	if refresh:
 		udev_add()

daemons/lvmdbusd/utils.py

@@ -10,11 +10,15 @@
 import xml.etree.ElementTree as Et
 import sys
 import inspect
+import collections
 import ctypes
+import errno
+import fcntl
 import os
+import stat
 import string
 import datetime
-from fcntl import fcntl, F_GETFL, F_SETFL
+import tempfile
 
 import dbus
 from lvmdbusd import cfg
@@ -280,17 +284,47 @@ def parse_tags(tags):
 	return dbus.Array([], signature='s')
 
 
-def _common_log(msg, *attributes):
-	cfg.stdout_lock.acquire()
+class DebugMessages(object):
+
+	def __init__(self, size=5000):
+		self.queue = collections.deque(maxlen=size)
+		self.lock = threading.RLock()
+
+	def add(self, message):
+		with self.lock:
+			self.queue.append(message)
+
+	def dump(self):
+		if cfg.args and not cfg.args.debug:
+			with self.lock:
+				if len(self.queue):
+					log_error("LVM dbus debug messages START last (%d max) messages" % self.queue.maxlen)
+					for m in self.queue:
+						print(m)
+					log_error("LVM dbus debug messages END")
+					self.queue.clear()
+
+
+def _format_log_entry(msg):
 	tid = ctypes.CDLL('libc.so.6').syscall(186)
 
-	if STDOUT_TTY:
+	if not cfg.systemd and STDOUT_TTY:
 		msg = "%s: %d:%d - %s" % \
 			(datetime.datetime.now().strftime("%b %d %H:%M:%S.%f"),
 			os.getpid(), tid, msg)
 
 	else:
-		msg = "%d:%d - %s" % (os.getpid(), tid, msg)
+		if cfg.systemd:
+			# Systemd already puts the daemon pid in the log, we'll just add the tid
+			msg = "[%d]: %s" % (tid, msg)
+		else:
+			msg = "[%d:%d]: %s" % (os.getpid(), tid, msg)
+	return msg
+
+
+def _common_log(msg, *attributes):
+	cfg.stdout_lock.acquire()
+	msg = _format_log_entry(msg)
 
 	if STDOUT_TTY and attributes:
 		print(color(msg, *attributes))
@@ -307,12 +341,19 @@ def _common_log(msg, *attributes):
 def log_debug(msg, *attributes):
 	if cfg.args and cfg.args.debug:
 		_common_log(msg, *attributes)
+	else:
+		if cfg.debug:
+			cfg.debug.add(_format_log_entry(msg))
 
 
 def log_error(msg, *attributes):
 	_common_log(msg, *attributes)
 
 
+def log_msg(msg, *attributes):
+	_common_log(msg, *attributes)
+
+
 def dump_threads_stackframe():
 	ident_to_name = {}
 
@@ -340,15 +381,32 @@ def dump_threads_stackframe():
 # noinspection PyUnusedLocal
 def handler(signum):
 	try:
+		# signal 10
 		if signum == signal.SIGUSR1:
+			cfg.debug.dump()
 			dump_threads_stackframe()
+		# signal 12
+		elif signum == signal.SIGUSR2:
+			cfg.debug.dump()
+			cfg.flightrecorder.dump()
 		else:
+			# If we are getting a SIGTERM, and we sent one to the lvm shell we
+			# will ignore this and keep running.
+			if signum == signal.SIGTERM and cfg.ignore_sigterm:
+				# Clear the flag, so we will exit on SIGTERM if we didn't
+				# send it.
+				cfg.ignore_sigterm = False
+				return True
+
+			# If lvm shell is in use, tell it to exit
+			if cfg.SHELL_IN_USE is not None:
+				cfg.SHELL_IN_USE.exit_shell()
 			cfg.run.value = 0
-			log_debug('Exiting daemon with signal %d' % signum)
+			log_error('Exiting daemon with signal %d' % signum)
 			if cfg.loop is not None:
 				cfg.loop.quit()
-	except:
-		st = traceback.format_exc()
+	except BaseException as be:
+		st = extract_stack_trace(be)
 		log_error("signal handler: exception (logged, not reported!) \n %s" % st)
 
 	# It's important we report that we handled the exception for the exception
@@ -572,6 +630,23 @@ def validate_tag(interface, tag):
 			% (tag, _ALLOWABLE_TAG_CH))
 
 
+def add_config_option(cmdline, key, value):
+	if 'help' in cmdline:
+		return cmdline
+
+	if key in cmdline:
+		for i, arg in enumerate(cmdline):
+			if arg == key:
+				if len(cmdline) <= i + 1:
+					raise dbus.exceptions.DBusException("Missing value for --config option.")
+				cmdline[i + 1] += " %s" % value
+				break
+	else:
+		cmdline.extend([key, value])
+
+	return cmdline
+
+
 def add_no_notify(cmdline):
 	"""
 	Given a command line to execute we will see if `--config` is present, if it
@@ -583,27 +658,18 @@ def add_no_notify(cmdline):
 	:rtype: list
 	"""
 
-	# Only after we have seen an external event will be disable lvm from sending
+	# Only after we have seen an external event will we disable lvm from sending
 	# us one when we call lvm
+	rv = cmdline
 	if cfg.got_external_event:
-		if 'help' in cmdline:
-			return cmdline
+		rv = add_config_option(rv, "--config", "global/notify_dbus=0")
 
-		if '--config' in cmdline:
-			for i, arg in enumerate(cmdline):
-				if arg == '--config':
-					if len(cmdline) <= i+1:
-						raise dbus.exceptions.DBusException("Missing value for --config option.")
-					cmdline[i+1] += " global/notify_dbus=0"
-					break
-		else:
-			cmdline.extend(['--config', 'global/notify_dbus=0'])
-	return cmdline
+	return rv
 
 
 # The methods below which start with mt_* are used to execute the desired code
-# on the the main thread of execution to alleviate any issues the dbus-python
-# library with regards to multi-threaded access.  Essentially, we are trying to
+# on the main thread of execution to alleviate any issues the dbus-python
+# library with regard to multithreaded access.  Essentially, we are trying to
 # ensure all dbus library interaction is done from the same thread!
 
 
@@ -617,9 +683,8 @@ def _async_handler(call_back, parameters):
 			call_back(*parameters)
 		else:
 			call_back()
-	except:
-		st = traceback.format_exc()
-		log_error("mt_async_call: exception (logged, not reported!) \n %s" % st)
+	except BaseException as be:
+		log_error("mt_async_call: exception (logged, not reported!) \n %s" % extract_stack_trace(be))
 
 
 # Execute the function on the main thread with the provided parameters, do
@@ -669,9 +734,6 @@ class MThreadRunner(object):
 				self.rc = self.f()
 		except BaseException as be:
 			self.exception = be
-			st = traceback.format_exc()
-			log_error("MThreadRunner: exception \n %s" % st)
-			log_error("Exception will be raised in calling thread!")
 
 
 def _remove_objects(dbus_objects_rm):
@@ -686,8 +748,8 @@ def mt_remove_dbus_objects(objs):
 
 # Make stream non-blocking
 def make_non_block(stream):
-	flags = fcntl(stream, F_GETFL)
-	fcntl(stream, F_SETFL, flags | os.O_NONBLOCK)
+	flags = fcntl.fcntl(stream, fcntl.F_GETFL)
+	fcntl.fcntl(stream, fcntl.F_SETFL, flags | os.O_NONBLOCK)
 
 
 def read_decoded(stream):
@@ -695,3 +757,122 @@ def read_decoded(stream):
 	if tmp:
 		return tmp.decode("utf-8")
 	return ''
+
+
+class LockFile(object):
+	"""
+	Simple lock file class
+	Based on Pg.1144 "The Linux Programming Interface" by Michael Kerrisk
+	"""
+	def __init__(self, lock_file):
+		self.fd = 0
+		self.lock_file = lock_file
+
+	def __enter__(self):
+		try:
+			self.fd = os.open(self.lock_file, os.O_CREAT | os.O_RDWR, stat.S_IRUSR | stat.S_IWUSR)
+
+			# Get and set the close on exec and lock the file
+			flags = fcntl.fcntl(self.fd, fcntl.F_GETFD)
+			flags |= fcntl.FD_CLOEXEC
+			fcntl.fcntl(self.fd, fcntl.F_SETFL, flags)
+			fcntl.lockf(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+		except OSError as e:
+			if e.errno == errno.EAGAIN:
+				log_error("Daemon already running, exiting!")
+			else:
+				log_error("Error during creation of lock file(%s): errno(%d), exiting!" % (self.lock_file, e.errno))
+			sys.exit(114)
+
+	def __exit__(self, _type, _value, _traceback):
+		os.close(self.fd)
+
+
+def extract_stack_trace(exception):
+	return ''.join(traceback.format_exception(None, exception, exception.__traceback__))
+
+
+def lvm_column_key(key):
+	# Check LV
+	if key.startswith("lv_") or key.startswith("vg_") or key.startswith("pool_") or \
+			key.endswith("_percent") or key.startswith("move_") or key.startswith("vdo_") or \
+			key in ["origin_uuid", "segtype", "origin", "data_lv", "metadata_lv"]:
+		return True
+	# Check VG
+	if key.startswith("vg_") or key.startswith("lv_") or key.startswith("pv_") or \
+			key in ["max_lv", "max_pv", "snap_count"]:
+		return True
+	# Check PV
+	if key.startswith("pv") or key.startswith("vg") or (key in ['dev_size', 'pe_start']):
+		return True
+	return False
+
+class LvmBug(RuntimeError):
+	"""
+	Things that are clearly a bug with lvm itself.
+	"""
+	def __init__(self, msg):
+		super().__init__(msg)
+
+	def __str__(self):
+		return "lvm bug encountered: %s" % ' '.join(self.args)
+
+
+class LvmDebugData:
+	def __init__(self):
+		self.fd = -1
+		self.fn = None
+
+	def _remove_file(self):
+		if self.fn is not None:
+			os.unlink(self.fn)
+			self.fn = None
+
+	def _close_fd(self):
+		if self.fd != -1:
+			os.close(self.fd)
+			self.fd = -1
+
+	def setup(self):
+		# Create a secure filename
+		self.fd, self.fn = tempfile.mkstemp(suffix=".log", prefix="lvmdbusd.lvm.debug.")
+		return self.fn
+
+	def lvm_complete(self):
+		# Remove the file ASAP, so we decrease our odds of leaving it
+		# around if the daemon gets killed by a signal -9
+		self._remove_file()
+
+	def dump(self):
+		# Read the file and log it to log_err
+		if self.fd != -1:
+			# How big could the verbose debug get?
+			debug = os.read(self.fd, 1024*1024*5)
+			debug_txt = debug.decode("utf-8")
+			for line in debug_txt.split("\n"):
+				log_error("lvm debug >>> %s" % line)
+			self._close_fd()
+		# In case lvm_complete doesn't get called.
+		self._remove_file()
+
+	def complete(self):
+		self._close_fd()
+		# In case lvm_complete doesn't get called.
+		self._remove_file()
+
+
+def get_error_msg(report_json):
+	# Get the error message from the returned JSON
+	if 'log' in report_json:
+		error_msg = ""
+		# Walk the entire log array and build an error string
+		for log_entry in report_json['log']:
+			if log_entry['log_type'] == "error":
+				if error_msg:
+					error_msg += ', ' + log_entry['log_message']
+				else:
+					error_msg = log_entry['log_message']
+
+		return error_msg
+
+	return None

daemons/lvmdbusd/vg.py

@@ -20,7 +20,7 @@ from .request import RequestEntry
 from .loader import common
 from .state import State
 from . import background
-from .utils import round_size, mt_remove_dbus_objects
+from .utils import round_size, mt_remove_dbus_objects, LvmBug, lvm_column_key
 from .job import JobState
 
 
@@ -31,17 +31,24 @@ def vgs_state_retrieve(selection, cache_refresh=True):
 	if cache_refresh:
 		cfg.db.refresh()
 
-	for v in cfg.db.fetch_vgs(selection):
-		rc.append(
-			VgState(
-				v['vg_uuid'], v['vg_name'], v['vg_fmt'], n(v['vg_size']),
-				n(v['vg_free']), v['vg_sysid'], n(v['vg_extent_size']),
-				n(v['vg_extent_count']), n(v['vg_free_count']),
-				v['vg_profile'], n(v['max_lv']), n(v['max_pv']),
-				n(v['pv_count']), n(v['lv_count']), n(v['snap_count']),
-				n(v['vg_seqno']), n(v['vg_mda_count']),
-				n(v['vg_mda_free']), n(v['vg_mda_size']),
-				n(v['vg_mda_used_count']), v['vg_attr'], v['vg_tags']))
+	try:
+		for v in cfg.db.fetch_vgs(selection):
+			rc.append(
+				VgState(
+					v['vg_uuid'], v['vg_name'], v['vg_fmt'], n(v['vg_size']),
+					n(v['vg_free']), v['vg_sysid'], n(v['vg_extent_size']),
+					n(v['vg_extent_count']), n(v['vg_free_count']),
+					v['vg_profile'], n(v['max_lv']), n(v['max_pv']),
+					n(v['pv_count']), n(v['lv_count']), n(v['snap_count']),
+					n(v['vg_seqno']), n(v['vg_mda_count']),
+					n(v['vg_mda_free']), n(v['vg_mda_size']),
+					n(v['vg_mda_used_count']), v['vg_attr'], v['vg_tags']))
+	except KeyError as ke:
+		# Sometimes lvm omits returning one of the keys we requested.
+		key = ke.args[0]
+		if lvm_column_key(key):
+			raise LvmBug("missing JSON key: '%s'" % key)
+		raise ke
 	return rc
 
 

daemons/lvmlockd/lvmlockd-core.c

@@ -5954,7 +5954,18 @@ static void adopt_locks(void)
 	}
 
 
-	/* FIXME: purge any remaining orphan locks in each rejoined ls? */
+	/* Try to purge the orphan locks when lock manager is dlm */
+	if (lm_support_dlm() && lm_is_running_dlm()) {
+		list_for_each_entry(ls, &ls_found, list) {
+			pthread_mutex_lock(&lockspaces_mutex);
+			ls1 = find_lockspace_name(ls->name);
+			if (ls1) {
+				log_debug("ls: %s purge locks", ls->name);
+				lm_purge_locks_dlm(ls1);
+			}
+			pthread_mutex_unlock(&lockspaces_mutex);
+		}
+	}
 
 	if (count_start_fail || count_adopt_fail)
 		goto fail;

daemons/lvmlockd/lvmlockd-dlm.c

@@ -220,6 +220,86 @@ int lm_prepare_lockspace_dlm(struct lockspace *ls)
 	return 0;
 }
 
+#define DLM_COMMS_PATH "/sys/kernel/config/dlm/cluster/comms"
+#define LOCK_LINE_MAX 1024
+static int get_local_nodeid(void)
+{
+	struct dirent *de;
+	DIR *ls_dir;
+	char ls_comms_path[PATH_MAX];
+	FILE *file = NULL;
+	char line[LOCK_LINE_MAX];
+	int rv = -1, val;
+
+	memset(ls_comms_path, 0, sizeof(ls_comms_path));
+	snprintf(ls_comms_path, PATH_MAX, "%s",DLM_COMMS_PATH);
+
+	if (!(ls_dir = opendir(ls_comms_path)))
+		return -ECONNREFUSED;
+
+	while ((de = readdir(ls_dir))) {
+		if (de->d_name[0] == '.')
+			continue;
+		memset(ls_comms_path, 0, sizeof(ls_comms_path));
+		snprintf(ls_comms_path, PATH_MAX, "%s/%s/local",
+		     DLM_COMMS_PATH, de->d_name);
+		file = fopen(ls_comms_path, "r");
+		if (!file)
+			continue;
+		if (fgets(line, LOCK_LINE_MAX, file)) {
+			fclose(file);
+			rv = sscanf(line, "%d", &val);
+			if ((rv == 1) && (val == 1 )) {
+				memset(ls_comms_path, 0, sizeof(ls_comms_path));
+				snprintf(ls_comms_path, PATH_MAX, "%s/%s/nodeid",
+				    DLM_COMMS_PATH, de->d_name);
+				file = fopen(ls_comms_path, "r");
+				if (!file)
+					continue;
+				if (fgets(line, LOCK_LINE_MAX, file)) {
+					rv = sscanf(line, "%d", &val);
+					if (rv == 1) {
+						fclose(file);
+						closedir(ls_dir);
+						return val;
+					}
+				}
+			}
+		}
+		fclose(file);
+	}
+
+	if (closedir(ls_dir))
+		log_error("get_local_nodeid closedir error");
+	return rv;
+}
+
+int lm_purge_locks_dlm(struct lockspace *ls)
+{
+	struct lm_dlm *lmd = (struct lm_dlm *)ls->lm_data;
+	int nodeid;
+	int rv = -1;
+
+	if (!lmd || !lmd->dh) {
+		log_error("purge_locks_dlm %s no dlm_handle_t error", ls->name);
+		goto fail;
+	}
+
+	nodeid = get_local_nodeid();
+	if (nodeid < 0) {
+		log_error("failed to get local nodeid");
+		goto fail;
+	}
+	if (dlm_ls_purge(lmd->dh, nodeid, 0)) {
+		log_error("purge_locks_dlm %s error", ls->name);
+		goto fail;
+	}
+
+	rv = 0;
+fail:
+	return rv;
+}
+
 int lm_add_lockspace_dlm(struct lockspace *ls, int adopt)
 {
 	struct lm_dlm *lmd = (struct lm_dlm *)ls->lm_data;

daemons/lvmlockd/lvmlockd-internal.h

@@ -392,6 +392,7 @@ static inline const char *mode_str(int x)
 int lm_init_vg_dlm(char *ls_name, char *vg_name, uint32_t flags, char *vg_args);
 int lm_prepare_lockspace_dlm(struct lockspace *ls);
 int lm_add_lockspace_dlm(struct lockspace *ls, int adopt);
+int lm_purge_locks_dlm(struct lockspace *ls);
 int lm_rem_lockspace_dlm(struct lockspace *ls, int free_vg);
 int lm_lock_dlm(struct lockspace *ls, struct resource *r, int ld_mode,
 		struct val_blk *vb_out, int adopt);
@@ -429,6 +430,11 @@ static inline int lm_add_lockspace_dlm(struct lockspace *ls, int adopt)
 	return -1;
 }
 
+static inline int lm_purge_locks_dlm(struct lockspace *ls)
+{
+	return -1;
+}
+
 static inline int lm_rem_lockspace_dlm(struct lockspace *ls, int free_vg)
 {
 	return -1;

daemons/lvmlockd/lvmlockd-sanlock.c

@@ -684,10 +684,10 @@ int lm_init_vg_sanlock(char *ls_name, char *vg_name, uint32_t flags, char *vg_ar
 			break;
 		}
 
-		if (rv) {
+		if (rv < 0) {
 			log_error("clear lv resource area %llu error %d",
 				  (unsigned long long)offset, rv);
-			break;
+			return rv;
 		}
 		offset += align_size;
 	}

daemons/lvmpolld/lvmpolld-core.c

@@ -52,7 +52,7 @@ static pthread_key_t key;
 
 static const char *_strerror_r(int errnum, struct lvmpolld_thread_data *data)
 {
-#ifdef _GNU_SOURCE
+#if defined(_GNU_SOURCE) && defined(STRERROR_R_CHAR_P)
 	return strerror_r(errnum, data->buf, sizeof(data->buf)); /* never returns NULL */
 #elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600)
 	return strerror_r(errnum, data->buf, sizeof(data->buf)) ? "" : data->buf;

device_mapper/Makefile

@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+# Copyright (C) 2018 - 2022 Red Hat, Inc. All rights reserved.
 #
 # This file is part of the device-mapper userspace tools.
 #
@@ -29,6 +29,7 @@ DEVICE_MAPPER_SOURCE=\
 	device_mapper/regex/parse_rx.c \
 	device_mapper/regex/ttree.c \
 	device_mapper/vdo/status.c \
+	device_mapper/vdo/vdo_reader.c \
 	device_mapper/vdo/vdo_target.c
 
 DEVICE_MAPPER_TARGET = device_mapper/libdevice-mapper.a

device_mapper/all.h

@@ -982,7 +982,9 @@ struct writecache_settings {
 	uint32_t fua;
 	uint32_t nofua;
 	uint32_t cleaner;
-	uint32_t max_age;
+	uint32_t max_age;         /* in milliseconds */
+	uint32_t metadata_only;
+	uint32_t pause_writeback; /* in milliseconds */
 
 	/*
 	 * Allow an unrecognized key and its val to be passed to the kernel for
@@ -1004,6 +1006,8 @@ struct writecache_settings {
 	unsigned nofua_set:1;
 	unsigned cleaner_set:1;
 	unsigned max_age_set:1;
+	unsigned metadata_only_set:1;
+	unsigned pause_writeback_set:1;
 };
 
 int dm_tree_node_add_writecache_target(struct dm_tree_node *node,
@@ -1049,6 +1053,7 @@ int dm_tree_node_add_integrity_target(struct dm_tree_node *node,
  */
 int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 				uint64_t size,
+				uint32_t vdo_version,
 				const char *vdo_pool_name,
 				const char *data_uuid,
 				uint64_t data_size,
@@ -1982,7 +1987,8 @@ struct dm_report_group;
 typedef enum {
 	DM_REPORT_GROUP_SINGLE,
 	DM_REPORT_GROUP_BASIC,
-	DM_REPORT_GROUP_JSON
+	DM_REPORT_GROUP_JSON,
+	DM_REPORT_GROUP_JSON_STD
 } dm_report_group_type_t;
 
 struct dm_report_group *dm_report_group_create(dm_report_group_type_t type, void *data);

device_mapper/ioctl/libdm-iface.c

@@ -139,7 +139,6 @@ static char *_align(char *ptr, unsigned int a)
 	return (char *) (((unsigned long) ptr + agn) & ~agn);
 }
 
-#ifdef DM_IOCTLS
 static unsigned _kernel_major = 0;
 static unsigned _kernel_minor = 0;
 static unsigned _kernel_release = 0;
@@ -182,6 +181,9 @@ int get_uname_version(unsigned *major, unsigned *minor, unsigned *release)
 
 	return 1;
 }
+
+#ifdef DM_IOCTLS
+
 /*
  * Set number to NULL to populate _dm_bitset - otherwise first
  * match is returned.

device_mapper/libdm-deptree.c

@@ -214,6 +214,7 @@ struct load_segment {
 	uint32_t device_id;		/* Thin */
 
 	// VDO params
+	uint32_t vdo_version;		/* VDO - version of target table line */
 	struct dm_tree_node *vdo_data;  /* VDO */
 	struct dm_vdo_target_params vdo_params; /* VDO */
 	const char *vdo_name;           /* VDO - device name is ALSO passed as table arg */
@@ -2678,6 +2679,10 @@ static int _writecache_emit_segment_line(struct dm_task *dmt,
 		count += 1;
 	if (seg->writecache_settings.max_age_set)
 		count += 2;
+	if (seg->writecache_settings.metadata_only_set)
+		count += 1;
+	if (seg->writecache_settings.pause_writeback_set)
+		count += 2;
 	if (seg->writecache_settings.new_key)
 		count += 2;
 
@@ -2729,6 +2734,14 @@ static int _writecache_emit_segment_line(struct dm_task *dmt,
 		EMIT_PARAMS(pos, " max_age %u", seg->writecache_settings.max_age);
 	}
 
+	if (seg->writecache_settings.metadata_only_set) {
+		EMIT_PARAMS(pos, " metadata_only");
+	}
+
+	if (seg->writecache_settings.pause_writeback_set) {
+		EMIT_PARAMS(pos, " pause_writeback %u", seg->writecache_settings.pause_writeback);
+	}
+
 	if (seg->writecache_settings.new_key) {
 		EMIT_PARAMS(pos, " %s %s",
 			seg->writecache_settings.new_key,
@@ -2849,13 +2862,18 @@ static int _thin_pool_emit_segment_line(struct dm_task *dmt,
 	return 1;
 }
 
-static int _vdo_emit_segment_line(struct dm_task *dmt,
+static int _vdo_emit_segment_line(struct dm_task *dmt, uint32_t major, uint32_t minor,
 				  struct load_segment *seg,
 				  char *params, size_t paramsize)
 {
 	int pos = 0;
 	char data[DM_FORMAT_DEV_BUFSIZE];
 	char data_dev[128]; // for /dev/dm-XXXX
+	uint64_t logical_blocks;
+	struct dm_task *vdo_dmt;
+	uint64_t start, length = 0;
+	char *type = NULL;
+	char *vdo_params = NULL;
 
 	if (!_build_dev_string(data, sizeof(data), seg->vdo_data))
 		return_0;
@@ -2865,18 +2883,59 @@ static int _vdo_emit_segment_line(struct dm_task *dmt,
 		return 0;
 	}
 
-	EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s "
-		    "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u",
-		    data_dev,
-		    seg->vdo_data_size / 8, // this parameter is in 4K units
-		    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
-		    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
-		    seg->vdo_params.block_map_era_length,
-		    seg->vdo_params.use_metadata_hints ? "on" : "off" ,
-		    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" :
-			(seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" :
-			(seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy
-		    seg->vdo_name,
+	/*
+	 * If there is already running VDO target, read 'existing' virtual size out of table line
+	 * and avoid reading it them from VDO metadata device
+	 *
+	 * NOTE: ATM VDO virtual size can be ONLY extended thus it's simple to recongnize 'right' size.
+	 * However if there would be supported also reduction, this check would need to check range.
+	 */
+	if ((vdo_dmt = dm_task_create(DM_DEVICE_TABLE))) {
+		if (dm_task_set_major(vdo_dmt, major) &&
+		    dm_task_set_minor(vdo_dmt, minor) &&
+		    dm_task_run(vdo_dmt)) {
+			(void) dm_get_next_target(vdo_dmt, NULL, &start, &length, &type, &vdo_params);
+			if (!type || strcmp(type, "vdo"))
+				length = 0;
+		}
+
+		dm_task_destroy(vdo_dmt);
+	}
+
+	if (!length && dm_vdo_parse_logical_size(data_dev, &logical_blocks))
+		length = logical_blocks * 8;
+
+	if (seg->size < length) {
+		log_debug_activation("Correcting VDO virtual volume size from " FMTu64  " to " FMTu64 ".",
+				     seg->size, length);
+		seg->size = length;
+	}
+
+	if (seg->vdo_version < 4) {
+		EMIT_PARAMS(pos, "V2 %s " FMTu64 " %u " FMTu64 " %u %s %s %s ",
+			    data_dev,
+			    seg->vdo_data_size / 8, // this parameter is in 4K units
+			    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
+			    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
+			    seg->vdo_params.block_map_era_length,
+			    seg->vdo_params.use_metadata_hints ? "on" : "off" ,
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_SYNC) ? "sync" :
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC) ? "async" :
+			    (seg->vdo_params.write_policy == DM_VDO_WRITE_POLICY_ASYNC_UNSAFE) ? "async-unsafe" : "auto", // policy
+			    seg->vdo_name);
+	} else {
+		EMIT_PARAMS(pos, "V4 %s " FMTu64 " %u " FMTu64 " %u "
+			    "deduplication %s compression %s ",
+			    data_dev,
+			    seg->vdo_data_size / 8, // this parameter is in 4K units
+			    seg->vdo_params.minimum_io_size * UINT32_C(512), //  sector to byte units
+			    seg->vdo_params.block_map_cache_size_mb * UINT64_C(256),	// 1MiB -> 4KiB units
+			    seg->vdo_params.block_map_era_length,
+			    seg->vdo_params.use_deduplication ? "on" : "off",
+			    seg->vdo_params.use_compression ? "on" : "off");
+	}
+
+	EMIT_PARAMS(pos, "maxDiscard %u ack %u bio %u bioRotationInterval %u cpu %u hash %u logical %u physical %u",
 		    seg->vdo_params.max_discard,
 		    seg->vdo_params.ack_threads,
 		    seg->vdo_params.bio_threads,
@@ -2951,7 +3010,7 @@ static int _emit_segment_line(struct dm_task *dmt, uint32_t major,
 		EMIT_PARAMS(pos, "%u %u ", seg->area_count, seg->stripe_size);
 		break;
 	case SEG_VDO:
-		if (!_vdo_emit_segment_line(dmt, seg, params, paramsize))
+		if (!_vdo_emit_segment_line(dmt, major, minor, seg, params, paramsize))
 		      return_0;
 		break;
 	case SEG_CRYPT:
@@ -4323,6 +4382,7 @@ void dm_tree_node_set_callback(struct dm_tree_node *dnode,
 
 int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 				uint64_t size,
+				uint32_t vdo_version,
 				const char *vdo_pool_name,
 				const char *data_uuid,
 				uint64_t data_size,
@@ -4344,11 +4404,13 @@ int dm_tree_node_add_vdo_target(struct dm_tree_node *node,
 	if (!_link_tree_nodes(node, seg->vdo_data))
 		return_0;
 
+	seg->vdo_version = vdo_version;
 	seg->vdo_params = *vtp;
 	seg->vdo_name = vdo_pool_name;
 	seg->vdo_data_size = data_size;
 
-	node->props.send_messages = 2;
+	if (seg->vdo_version < 4)
+		node->props.send_messages = 2;
 
 	return 1;
 }

device_mapper/libdm-report.c

@@ -384,172 +384,246 @@ int dm_report_field_percent(struct dm_report *rh,
 	return 1;
 }
 
-struct str_list_sort_value_item {
+struct pos_len {
 	unsigned pos;
 	size_t len;
 };
 
+struct str_pos_len {
+	const char *str;
+	struct pos_len item;
+};
+
 struct str_list_sort_value {
 	const char *value;
-	struct str_list_sort_value_item *items;
+	struct pos_len *items;
 };
 
-struct str_list_sort_item {
-	const char *str;
-	struct str_list_sort_value_item item;
-};
-
-static int _str_list_sort_item_cmp(const void *a, const void *b)
+static int _str_sort_cmp(const void *a, const void *b)
 {
-	const struct str_list_sort_item *slsi_a = (const struct str_list_sort_item *) a;
-	const struct str_list_sort_item *slsi_b = (const struct str_list_sort_item *) b;
-
-	return strcmp(slsi_a->str, slsi_b->str);
+	return strcmp(((const struct str_pos_len *) a)->str, ((const struct str_pos_len *) b)->str);
 }
 
+#define FIELD_STRING_LIST_DEFAULT_DELIMITER ","
+
 static int _report_field_string_list(struct dm_report *rh,
 				     struct dm_report_field *field,
 				     const struct dm_list *data,
 				     const char *delimiter,
-				     int sort)
+				     int sort_repstr)
 {
-	static const char _string_list_grow_object_failed_msg[] = "dm_report_field_string_list: dm_pool_grow_object_failed";
-	struct str_list_sort_value *sort_value = NULL;
-	unsigned int list_size, pos, i;
-	struct str_list_sort_item *arr = NULL;
+	static const char _error_msg_prefix[] = "_report_field_string_list: ";
+	unsigned int list_size, i, pos;
+	struct str_pos_len *arr = NULL;
 	struct dm_str_list *sl;
-	size_t delimiter_len, len;
-	void *object;
+	size_t delimiter_len, repstr_str_len, repstr_size;
+	char *repstr = NULL;
+	struct pos_len *repstr_extra;
+	struct str_list_sort_value *sortval = NULL;
 	int r = 0;
 
-	if (!(sort_value = dm_pool_zalloc(rh->mem, sizeof(struct str_list_sort_value)))) {
-		log_error("dm_report_field_string_list: dm_pool_zalloc failed for sort_value");
-		return 0;
-	}
+	/*
+	 * The 'field->report_string' has 2 parts:
+	 *
+	 *    - string representing the whole string list
+	 *      (terminated by '\0' at its end as usual)
+	 *
+	 *    - extra info beyond the end of the string representing
+	 *      position and length of each list item within the
+	 *      field->report_string (array of 'struct pos_len')
+	 *
+	 * We can use the extra info to unambiguously identify list items,
+	 * the delimiter is not enough here as it's not assured it won't appear
+	 * in list item itself. We will make use of this extra info in case
+	 * we need to apply further formatting to the list in dm_report_output
+	 * where the pure field->report_string is not enough for printout.
+	 *
+	 *
+	 * The 'field->sort_value' contains a value of type 'struct
+	 * str_list_sort_value' ('sortval'). This one has a pointer to the
+	 * 'field->report_string' string ('sortval->value') and info
+	 * about position and length of each list item within the string
+	 * (array of 'struct pos_len').
+	 *
+	 *
+	 * The 'field->report_string' is either in sorted or unsorted form,
+	 * depending on 'sort_repstr' arg.
+	 *
+	 * The 'field->sort_value.items' is always in sorted form because
+	 * we need that for effective sorting and selection.
+	 *
+	 * If 'field->report_string' is sorted, then field->report_string
+	 * and field->sort_value.items share the same array of
+	 * 'struct pos_len' (because they're both sorted the same way),
+	 * otherwise, each one has its own array.
+	 *
+	 * The very first item in the array of 'struct pos_len' is always
+	 * a pair denoting '[list_size,strlen(field->report_string)]'. The
+	 * rest of items denote start and lenght of each item in the list.
+         *
+	 *
+	 * For example, if we have a list with "abc", "xy", "defgh"
+	 * as input and delimiter is ",", we end up with either:
+	 *
+	 *  A) if we don't want the report string sorted ('sort_repstr == 0'):
+	 *
+	 *    - field->report_string = repstr
+	 *
+	 *       repstr      repstr_extra
+	 *         |             |
+	 *         V             V
+	 *         abc,xy,defgh\0{[3,12],[0,3],[4,2],[7,5]}
+	 *         |____________||________________________|
+	 *             string     array of struct pos_len
+	 *                        |____||________________|
+	 *                        #items      items
+	 *
+	 *    - field->sort_value = sortval
+	 *
+	 *       sortval->value = repstr
+	 *       sortval->items = {[3,12],[0,3],[7,5],[4,2]}
+	 *                         (that is 'abc,defgh,xy')
+	 *
+	 *
+	 *  B) if we want the report string sorted ('sort_repstr == 1'):
+	 *
+	 *    - field->report_string = repstr
+	 *
+	 *       repstr     repstr_extra
+	 *         |             |
+	 *         V             V
+	 *         abc,defgh,xy\0{[3,12],[0,3],[4,5],[10,2]}
+	 *         |____________||________________________|
+	 *             string     array of struct pos_len
+	 *                        |____||________________|
+	 *                        #items      items
+	 *
+	 *     - field->sort_value = sortval
+	 *
+	 *       sortval->value = repstr
+	 *       sortval->items = repstr_extra
+	 *                       (that is 'abc,defgh,xy')
+	 */
 
+	if (!delimiter)
+		delimiter = FIELD_STRING_LIST_DEFAULT_DELIMITER;
+	delimiter_len = strlen(delimiter);
 	list_size = dm_list_size(data);
 
-	/*
-	 * Sort value stores the pointer to the report_string and then
-	 * position and length for each list element withing the report_string.
-	 * The first element stores number of elements in 'len' (therefore
-	 * list_size + 1 is used below for the extra element).
-	 * For example, with this input:
-	 *   sort = 0;  (we don't want to report sorted)
-	 *   report_string = "abc,xy,defgh";  (this is reported)
-	 *
-	 * ...we end up with:
-	 *   sort_value->value = report_string; (we'll use the original report_string for indices)
-	 *   sort_value->items[0] = {0,3};  (we have 3 items)
-	 *   sort_value->items[1] = {0,3};  ("abc")
-	 *   sort_value->items[2] = {7,5};  ("defgh")
-	 *   sort_value->items[3] = {4,2};  ("xy")
-	 *
-	 *   The items alone are always sorted while in report_string they can be
-	 *   sorted or not (based on "sort" arg) - it depends on how we prefer to
-	 *   display the list. Having items sorted internally helps with searching
-	 *   through them.
-	 */
-	if (!(sort_value->items = dm_pool_zalloc(rh->mem, (list_size + 1) * sizeof(struct str_list_sort_value_item)))) {
-		log_error("dm_report_fiel_string_list: dm_pool_zalloc failed for sort value items");
+	if (!(sortval = dm_pool_alloc(rh->mem, sizeof(struct str_list_sort_value)))) {
+		log_error("%s failed to allocate sort value structure", _error_msg_prefix);
 		goto out;
 	}
-	sort_value->items[0].len = list_size;
 
 	/* zero items */
-	if (!list_size) {
-		sort_value->value = field->report_string = "";
-		field->sort_value = sort_value;
+	if (list_size == 0) {
+		field->report_string = sortval->value = "";
+		sortval->items = NULL;
+		field->sort_value = sortval;
 		return 1;
 	}
 
 	/* one item */
 	if (list_size == 1) {
 		sl = (struct dm_str_list *) dm_list_first(data);
-		if (!sl ||
-		    !(sort_value->value = field->report_string = dm_pool_strdup(rh->mem, sl->str))) {
-			log_error("dm_report_field_string_list: dm_pool_strdup failed");
+
+		repstr_str_len = strlen(sl->str);
+		repstr_size = repstr_str_len + 1 + (2 * sizeof(struct pos_len));
+
+		if (!(repstr = dm_pool_alloc(rh->mem, repstr_size))) {
+			log_error("%s failed to allocate report string structure", _error_msg_prefix);
 			goto out;
 		}
-		sort_value->items[1].pos = 0;
-		sort_value->items[1].len = strlen(sl->str);
-		field->sort_value = sort_value;
+		repstr_extra = (struct pos_len *) (repstr + repstr_str_len + 1);
+
+		memcpy(repstr, sl->str, repstr_str_len + 1);
+		memcpy(repstr_extra, &((struct pos_len) {.pos = 1, .len = repstr_str_len}), sizeof(struct pos_len));
+		memcpy(repstr_extra + 1, &((struct pos_len) {.pos = 0, .len = repstr_str_len}), sizeof(struct pos_len));
+
+		sortval->value = field->report_string = repstr;
+		sortval->items = repstr_extra;
+		field->sort_value = sortval;
 		return 1;
 	}
 
-	/* more than one item - sort the list */
-	if (!(arr = malloc(sizeof(struct str_list_sort_item) * list_size))) {
-		log_error("dm_report_field_string_list: malloc failed");
+	/* more than one item - allocate temporary array for string list items for further processing */
+	if (!(arr = malloc(list_size * sizeof(struct str_pos_len)))) {
+		log_error("%s failed to allocate temporary array for processing", _error_msg_prefix);
 		goto out;
 	}
 
-	if (!(dm_pool_begin_object(rh->mem, 256))) {
-		log_error(_string_list_grow_object_failed_msg);
-		goto out;
-	}
-
-	if (!delimiter)
-		delimiter = ",";
-	delimiter_len = strlen(delimiter);
-
-	i = pos = 0;
+	i = 0;
+	repstr_size = 0;
 	dm_list_iterate_items(sl, data) {
 		arr[i].str = sl->str;
-		if (!sort) {
-			/* sorted outpud not required - report the list as it is */
-			len = strlen(sl->str);
-			if (!dm_pool_grow_object(rh->mem, arr[i].str, len) ||
-			    (i+1 != list_size && !dm_pool_grow_object(rh->mem, delimiter, delimiter_len))) {
-				log_error(_string_list_grow_object_failed_msg);
-				goto out;
-			}
-			arr[i].item.pos = pos;
-			arr[i].item.len = len;
-			pos = i+1 == list_size ? pos+len : pos+len+delimiter_len;
-		}
+		repstr_size += (arr[i].item.len = strlen(sl->str));
 		i++;
 	}
 
-	qsort(arr, i, sizeof(struct str_list_sort_item), _str_list_sort_item_cmp);
+	/*
+	 * At this point, repstr_size contains sum of lengths of all string list items.
+	 * Now, add these to the repstr_size:
+	 *
+	 * 	--> sum of character count used by all delimiters: + ((list_size - 1) * delimiter_len)
+	 *
+	 * 	--> '\0' used at the end of the string list: + 1
+	 *
+	 * 	--> sum of structures used to keep info about pos and length of each string list item:
+	 * 	    [0, <list_size>] [<pos1>,<size1>] [<pos2>,<size2>] ...
+	 * 	    That is: + ((list_size + 1) * sizeof(struct pos_len))
+	 */
+	repstr_size += ((list_size - 1) * delimiter_len);
+	repstr_str_len = repstr_size;
+	repstr_size += 1 + ((list_size + 1) * sizeof(struct pos_len));
 
-	for (i = 0, pos = 0; i < list_size; i++) {
-		if (sort) {
-			/* sorted output required - report the list as sorted */
-			len = strlen(arr[i].str);
-			if (!dm_pool_grow_object(rh->mem, arr[i].str, len) ||
-			    (i+1 != list_size && !dm_pool_grow_object(rh->mem, delimiter, delimiter_len))) {
-				log_error(_string_list_grow_object_failed_msg);
-				goto out;
-			}
-			/*
-			 * Save position and length of the string
-			 * element in report_string for sort_value.
-			 * Use i+1 here since items[0] stores list size!!!
-			 */
-			sort_value->items[i+1].pos = pos;
-			sort_value->items[i+1].len = len;
-			pos = i+1 == list_size ? pos+len : pos+len+delimiter_len;
-		} else {
-			sort_value->items[i+1].pos = arr[i].item.pos;
-			sort_value->items[i+1].len = arr[i].item.len;
-		}
-	}
+	if (sort_repstr)
+		qsort(arr, list_size, sizeof(struct str_pos_len), _str_sort_cmp);
 
-	if (!dm_pool_grow_object(rh->mem, "\0", 1)) {
-		log_error(_string_list_grow_object_failed_msg);
+	if (!(repstr = dm_pool_alloc(rh->mem, repstr_size))) {
+		log_error("%s failed to allocate report string structure", _error_msg_prefix);
 		goto out;
 	}
+	repstr_extra = (struct pos_len *) (repstr + repstr_str_len + 1);
 
-	object = dm_pool_end_object(rh->mem);
-	sort_value->value = object;
-	field->sort_value = sort_value;
-	field->report_string = object;
+	memcpy(repstr_extra, &(struct pos_len) {.pos = list_size, .len = repstr_str_len}, sizeof(struct pos_len));
+	for (i = 0, pos = 0; i < list_size; i++) {
+		arr[i].item.pos = pos;
+
+		memcpy(repstr + pos, arr[i].str, arr[i].item.len);
+		memcpy(repstr_extra + i + 1, &arr[i].item, sizeof(struct pos_len));
+
+		pos += arr[i].item.len;
+		if (i + 1 < list_size) {
+			memcpy(repstr + pos, delimiter, delimiter_len);
+			pos += delimiter_len;
+		}
+	}
+	*(repstr + pos) = '\0';
+
+	sortval->value = repstr;
+	if (sort_repstr)
+		sortval->items = repstr_extra;
+	else {
+		if (!(sortval->items = dm_pool_alloc(rh->mem, (list_size + 1) * sizeof(struct pos_len)))) {
+			log_error("%s failed to allocate array of items inside sort value structure",
+				  _error_msg_prefix);
+			goto out;
+		}
+
+		qsort(arr, list_size, sizeof(struct str_pos_len), _str_sort_cmp);
+
+		sortval->items[0] = (struct pos_len) {.pos = list_size, .len = repstr_str_len};
+		for (i = 0; i < list_size; i++)
+			sortval->items[i+1] = arr[i].item;
+	}
+
+	field->report_string = repstr;
+	field->sort_value = sortval;
 	r = 1;
 out:
-	if (!r && sort_value)
-		dm_pool_free(rh->mem, sort_value);
+	if (!r && sortval)
+		dm_pool_free(rh->mem, sortval);
 	free(arr);
-
 	return r;
 }
 
@@ -1688,7 +1762,7 @@ static int _cmp_field_string_list_strict_all(const struct str_list_sort_value *v
 	struct dm_str_list *sel_item;
 	unsigned int i = 1;
 
-	if (!val->items[0].len) {
+	if (!val->items) {
 		if (sel_list_size == 1) {
 			/* match blank string list with selection defined as blank string only */
 			sel_item = dm_list_item(dm_list_first(&sel->str_list.list), struct dm_str_list);
@@ -1698,7 +1772,7 @@ static int _cmp_field_string_list_strict_all(const struct str_list_sort_value *v
 	}
 
 	/* if item count differs, it's clear the lists do not match */
-	if (val->items[0].len != sel_list_size)
+	if (val->items[0].pos != sel_list_size)
 		return 0;
 
 	/* both lists are sorted so they either match 1:1 or not */
@@ -1721,7 +1795,7 @@ static int _cmp_field_string_list_subset_all(const struct str_list_sort_value *v
 	unsigned int i, last_found = 1;
 	int r = 0;
 
-	if (!val->items[0].len) {
+	if (!val->items) {
 		if (sel_list_size == 1) {
 			/* match blank string list with selection defined as blank string only */
 			sel_item = dm_list_item(dm_list_first(&sel->str_list.list), struct dm_str_list);
@@ -1733,7 +1807,7 @@ static int _cmp_field_string_list_subset_all(const struct str_list_sort_value *v
 	/* check selection is a subset of the value */
 	dm_list_iterate_items(sel_item, &sel->str_list.list) {
 		r = 0;
-		for (i = last_found; i <= val->items[0].len; i++) {
+		for (i = last_found; i <= val->items[0].pos; i++) {
 			if ((strlen(sel_item->str) == val->items[i].len) &&
 			    !strncmp(sel_item->str, val->value + val->items[i].pos, val->items[i].len)) {
 				last_found = i;
@@ -1755,7 +1829,7 @@ static int _cmp_field_string_list_any(const struct str_list_sort_value *val,
 	unsigned int i;
 
 	/* match blank string list with selection that contains blank string */
-	if (!val->items[0].len) {
+	if (!val->items) {
 		dm_list_iterate_items(sel_item, &sel->str_list.list) {
 			if (!strcmp(sel_item->str, ""))
 				return 1;
@@ -1768,7 +1842,7 @@ static int _cmp_field_string_list_any(const struct str_list_sort_value *val,
 		 * TODO: Optimize this so we don't need to compare the whole lists' content.
 		 *       Make use of the fact that the lists are sorted!
 		 */
-		for (i = 1; i <= val->items[0].len; i++) {
+		for (i = 1; i <= val->items[0].pos; i++) {
 			if ((strlen(sel_item->str) == val->items[i].len) &&
 			    !strncmp(sel_item->str, val->value + val->items[i].pos, val->items[i].len))
 				return 1;
@@ -4360,6 +4434,7 @@ static int _sort_rows(struct dm_report *rh)
 #define JSON_ARRAY_START       "["
 #define JSON_ARRAY_END         "]"
 #define JSON_ESCAPE_CHAR       "\\"
+#define JSON_NULL              "null"
 
 #define UNABLE_TO_EXTEND_OUTPUT_LINE_MSG "dm_report: Unable to extend output line"
 
@@ -4369,38 +4444,42 @@ static int _is_basic_report(struct dm_report *rh)
 	       (rh->group_item->group->type == DM_REPORT_GROUP_BASIC);
 }
 
+static int _is_json_std_report(struct dm_report *rh)
+{
+	return rh->group_item &&
+	       rh->group_item->group->type == DM_REPORT_GROUP_JSON_STD;
+}
+
 static int _is_json_report(struct dm_report *rh)
 {
 	return rh->group_item &&
-	       (rh->group_item->group->type == DM_REPORT_GROUP_JSON);
+	       (rh->group_item->group->type == DM_REPORT_GROUP_JSON ||
+		rh->group_item->group->type == DM_REPORT_GROUP_JSON_STD);
 }
 
-/*
- * Produce report output
- */
-static int _output_field(struct dm_report *rh, struct dm_report_field *field)
+static int _is_pure_numeric_field(struct dm_report_field *field)
+{
+	return field->props->flags & (DM_REPORT_FIELD_TYPE_NUMBER | DM_REPORT_FIELD_TYPE_PERCENT);
+}
+
+static const char *_get_field_id(struct dm_report *rh, struct dm_report_field *field)
 {
 	const struct dm_report_field_type *fields = field->props->implicit ? _implicit_report_fields
 									   : rh->fields;
+
+	return fields[field->props->field_num].id;
+}
+
+static int _output_field_basic_fmt(struct dm_report *rh, struct dm_report_field *field)
+{
 	char *field_id;
 	int32_t width;
 	uint32_t align;
-	const char *repstr;
-	const char *p1_repstr, *p2_repstr;
 	char *buf = NULL;
 	size_t buf_size = 0;
 
-	if (_is_json_report(rh)) {
-		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
-		    !dm_pool_grow_object(rh->mem, fields[field->props->field_num].id, 0) ||
-		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
-		    !dm_pool_grow_object(rh->mem, JSON_PAIR, 1) ||
-		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-			log_error("dm_report: Unable to extend output line");
-			return 0;
-		}
-	} else if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
-		if (!(field_id = strdup(fields[field->props->field_num].id))) {
+	if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
+		if (!(field_id = strdup(_get_field_id(rh, field)))) {
 			log_error("dm_report: Failed to copy field name");
 			return 0;
 		}
@@ -4431,43 +4510,14 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 		}
 	}
 
-	repstr = field->report_string;
-	width = field->props->width;
-	if (!(rh->flags & DM_REPORT_OUTPUT_ALIGNED)) {
-		if (_is_json_report(rh)) {
-			/* Escape any JSON_QUOTE that may appear in reported string. */
-			p1_repstr = repstr;
-			while ((p2_repstr = strstr(p1_repstr, JSON_QUOTE))) {
-				if (p2_repstr > p1_repstr) {
-					if (!dm_pool_grow_object(rh->mem, p1_repstr, p2_repstr - p1_repstr)) {
-						log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-						return 0;
-					}
-				}
-				if (!dm_pool_grow_object(rh->mem, JSON_ESCAPE_CHAR, 1) ||
-				    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-					log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-					return 0;
-				}
-				p1_repstr = p2_repstr + 1;
-			}
-
-			if (!dm_pool_grow_object(rh->mem, p1_repstr, 0)) {
-				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-				return 0;
-			}
-		} else {
-			if (!dm_pool_grow_object(rh->mem, repstr, 0)) {
-				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-				return 0;
-			}
-		}
-	} else {
+	if (rh->flags & DM_REPORT_OUTPUT_ALIGNED) {
 		if (!(align = field->props->flags & DM_REPORT_FIELD_ALIGN_MASK))
 			align = ((field->props->flags & DM_REPORT_FIELD_TYPE_NUMBER) ||
-				 (field->props->flags & DM_REPORT_FIELD_TYPE_SIZE)) ? 
+				 (field->props->flags & DM_REPORT_FIELD_TYPE_SIZE)) ?
 				DM_REPORT_FIELD_ALIGN_RIGHT : DM_REPORT_FIELD_ALIGN_LEFT;
 
+		width = field->props->width;
+
 		/* Including trailing '\0'! */
 		buf_size = width + 1;
 		if (!(buf = malloc(buf_size))) {
@@ -4477,7 +4527,7 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 
 		if (align & DM_REPORT_FIELD_ALIGN_LEFT) {
 			if (dm_snprintf(buf, buf_size, "%-*.*s",
-					 width, width, repstr) < 0) {
+					 width, width, field->report_string) < 0) {
 				log_error("dm_report: left-aligned snprintf() failed");
 				goto bad;
 			}
@@ -4487,7 +4537,7 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 			}
 		} else if (align & DM_REPORT_FIELD_ALIGN_RIGHT) {
 			if (dm_snprintf(buf, buf_size, "%*.*s",
-					 width, width, repstr) < 0) {
+					 width, width, field->report_string) < 0) {
 				log_error("dm_report: right-aligned snprintf() failed");
 				goto bad;
 			}
@@ -4496,6 +4546,11 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 				goto bad;
 			}
 		}
+	} else {
+		if (!dm_pool_grow_object(rh->mem, field->report_string, 0)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
 	}
 
 	if (rh->flags & DM_REPORT_OUTPUT_FIELD_NAME_PREFIX) {
@@ -4505,21 +4560,164 @@ static int _output_field(struct dm_report *rh, struct dm_report_field *field)
 				goto bad;
 			}
 		}
-	} else if (_is_json_report(rh)) {
-		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
-			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
-			goto bad;
-		}
 	}
 
 	free(buf);
 	return 1;
-
 bad:
 	free(buf);
 	return 0;
 }
 
+static int _safe_repstr_output(struct dm_report *rh, const char *repstr, size_t len)
+{
+	const char *p_repstr;
+	const char *repstr_end = len ? repstr + len : repstr + strlen(repstr);
+
+	/* Escape any JSON_QUOTE that may appear in reported string. */
+	while (1) {
+		if (!(p_repstr = memchr(repstr, JSON_QUOTE[0], repstr_end - repstr)))
+			break;
+
+		if (p_repstr > repstr) {
+			if (!dm_pool_grow_object(rh->mem, repstr, p_repstr - repstr)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+		}
+		if (!dm_pool_grow_object(rh->mem, JSON_ESCAPE_CHAR, 1) ||
+		    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+		repstr = p_repstr + 1;
+	}
+
+	if (!dm_pool_grow_object(rh->mem, repstr, repstr_end - repstr)) {
+		log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int _output_field_json_fmt(struct dm_report *rh, struct dm_report_field *field)
+{
+	const char *repstr;
+	size_t list_size, i;
+	struct pos_len *pos_len;
+
+	if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
+	    !dm_pool_grow_object(rh->mem, _get_field_id(rh, field),  0) ||
+	    !dm_pool_grow_object(rh->mem, JSON_QUOTE, 1) ||
+	    !dm_pool_grow_object(rh->mem, JSON_PAIR, 1)) {
+		log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+		return 0;
+	}
+
+	if (field->props->flags & DM_REPORT_FIELD_TYPE_STRING_LIST) {
+		if (!_is_json_std_report(rh)) {
+
+			/* string list in JSON - report whole list as simple string in quotes */
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			if (!_safe_repstr_output(rh, field->report_string, 0))
+				return_0;
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			return 1;
+		}
+
+		/* string list in JSON_STD - report list as proper JSON array */
+
+		if (!dm_pool_grow_object(rh->mem, JSON_ARRAY_START, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+
+		if (*field->report_string != 0) {
+			pos_len = (struct pos_len *) (field->report_string +
+				  ((struct str_list_sort_value *) field->sort_value)->items[0].len + 1);
+			list_size = pos_len->pos;
+		} else
+			list_size = 0;
+
+		for (i = 0; i < list_size; i++) {
+			pos_len++;
+
+			if (i != 0) {
+				if (!dm_pool_grow_object(rh->mem, JSON_SEPARATOR, 1)) {
+					log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+					return 0;
+				}
+			}
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+
+			if (!_safe_repstr_output(rh, field->report_string + pos_len->pos, pos_len->len))
+				return_0;
+
+			if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+				log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+				return 0;
+			}
+		}
+
+		if (!dm_pool_grow_object(rh->mem, JSON_ARRAY_END, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+
+		return 1;
+	}
+
+	/* all other types than string list - handle both JSON and JSON_STD */
+
+	if (!(_is_json_std_report(rh) && _is_pure_numeric_field(field))) {
+		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+	}
+
+	if (_is_json_std_report(rh) && _is_pure_numeric_field(field) && !*field->report_string)
+		repstr = JSON_NULL;
+	else
+		repstr = field->report_string;
+
+	if (!_safe_repstr_output(rh, repstr, 0))
+		return_0;
+
+	if (!(_is_json_std_report(rh) && _is_pure_numeric_field(field))) {
+		if (!dm_pool_grow_object(rh->mem, JSON_QUOTE, 1)) {
+			log_error(UNABLE_TO_EXTEND_OUTPUT_LINE_MSG);
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
+/*
+ * Produce report output
+ */
+static int _output_field(struct dm_report *rh, struct dm_report_field *field)
+{
+	return _is_json_report(rh) ? _output_field_json_fmt(rh, field)
+				   : _output_field_basic_fmt(rh, field);
+}
+
 static void _destroy_rows(struct dm_report *rh)
 {
 	/*
@@ -4983,6 +5181,7 @@ int dm_report_group_push(struct dm_report_group *group, struct dm_report *report
 				goto_bad;
 			break;
 		case DM_REPORT_GROUP_JSON:
+		case DM_REPORT_GROUP_JSON_STD:
 			if (!_report_group_push_json(item, data))
 				goto_bad;
 			break;
@@ -5046,6 +5245,7 @@ int dm_report_group_pop(struct dm_report_group *group)
 				return_0;
 			break;
 		case DM_REPORT_GROUP_JSON:
+		case DM_REPORT_GROUP_JSON_STD:
 			if (!_report_group_pop_json(item))
 				return_0;
 			break;
@@ -5082,7 +5282,7 @@ int dm_report_group_output_and_pop_all(struct dm_report_group *group)
 			return_0;
 	}
 
-	if (group->type == DM_REPORT_GROUP_JSON) {
+	if (group->type == DM_REPORT_GROUP_JSON || group->type == DM_REPORT_GROUP_JSON_STD) {
 		_json_output_start(group);
 		log_print(JSON_OBJECT_END);
 		group->indent -= JSON_INDENT_UNIT;

device_mapper/vdo/target.h

@@ -77,8 +77,10 @@ enum dm_vdo_write_policy {
 struct dm_vdo_target_params {
 	uint32_t minimum_io_size;       // in sectors
 	uint32_t block_map_cache_size_mb;
-	uint32_t block_map_era_length;	// format period
-
+	union {
+		uint32_t block_map_era_length;	// format period
+		uint32_t block_map_period;      // supported alias
+	};
 	uint32_t check_point_frequency;
 	uint32_t index_memory_size_mb;  // format
 
@@ -106,6 +108,8 @@ struct dm_vdo_target_params {
 bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 				   uint64_t vdo_size);
 
+bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks);
+
 //----------------------------------------------------------------
 
 #endif

device_mapper/vdo/vdo_limits.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2018-2022 Red Hat, Inc. All rights reserved.
  *
  * This file is part of the device-mapper userspace tools.
  *
@@ -15,49 +15,52 @@
 #ifndef DEVICE_MAPPER_VDO_LIMITS_H
 #define DEVICE_MAPPER_VDO_LIMITS_H
 
+#ifndef SECTOR_SHIFT
+#define SECTOR_SHIFT 9L
+#endif
+
 #define DM_VDO_BLOCK_SIZE			UINT64_C(8)		// 4KiB in sectors
+#define DM_VDO_BLOCK_SIZE_KB			(DM_VDO_BLOCK_SIZE << SECTOR_SHIFT)
 
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB	(128)			// 128MiB
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB	(16 * 1024 * 1024 - 1)	// 16TiB - 1
 #define DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_PER_LOGICAL_THREAD  (4096 * DM_VDO_BLOCK_SIZE_KB)
 
-#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM	(1)
-#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM	(16380)
+#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM	1
+#define DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM	16380
 
-#define DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB	(256)			// 0.25 GiB
+#define DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB	256			// 0.25 GiB
 #define DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB	(1024 * 1024 * 1024)	// 1TiB
 
-//#define DM_VDO_READ_CACHE_SIZE_MINIMUM_MB	(0)
-#define DM_VDO_READ_CACHE_SIZE_MAXIMUM_MB	(16 * 1024 * 1024 - 1)	// 16TiB - 1
-
-#define DM_VDO_SLAB_SIZE_MINIMUM_MB		(128)			// 128MiB
+#define DM_VDO_SLAB_SIZE_MINIMUM_MB		128			// 128MiB
 #define DM_VDO_SLAB_SIZE_MAXIMUM_MB		(32 * 1024)		// 32GiB
+#define DM_VDO_SLABS_MAXIMUM			8192
 
-//#define DM_VDO_LOGICAL_SIZE_MINIMUM_MB	(0)
-#define DM_VDO_LOGICAL_SIZE_MAXIMUM_MB	(UINT64_C(4) * 1024 * 1024 * 1024) // 4PiB
+#define DM_VDO_LOGICAL_SIZE_MAXIMUM	(UINT64_C(4) * 1024 * 1024 * 1024 * 1024 * 1024 >> SECTOR_SHIFT) // 4PiB
+#define DM_VDO_PHYSICAL_SIZE_MAXIMUM	(UINT64_C(64) * DM_VDO_BLOCK_SIZE_KB * 1024 * 1024 * 1024 >> SECTOR_SHIFT) // 256TiB
 
-//#define DM_VDO_ACK_THREADS_MINIMUM		(0)
-#define DM_VDO_ACK_THREADS_MAXIMUM		(100)
+#define DM_VDO_ACK_THREADS_MINIMUM		0
+#define DM_VDO_ACK_THREADS_MAXIMUM		100
 
-#define DM_VDO_BIO_THREADS_MINIMUM		(1)
-#define DM_VDO_BIO_THREADS_MAXIMUM		(100)
+#define DM_VDO_BIO_THREADS_MINIMUM		1
+#define DM_VDO_BIO_THREADS_MAXIMUM		100
 
-#define DM_VDO_BIO_ROTATION_MINIMUM		(1)
-#define DM_VDO_BIO_ROTATION_MAXIMUM		(1024)
+#define DM_VDO_BIO_ROTATION_MINIMUM		1
+#define DM_VDO_BIO_ROTATION_MAXIMUM		1024
 
-#define DM_VDO_CPU_THREADS_MINIMUM		(1)
-#define DM_VDO_CPU_THREADS_MAXIMUM		(100)
+#define DM_VDO_CPU_THREADS_MINIMUM		1
+#define DM_VDO_CPU_THREADS_MAXIMUM		100
 
-//#define DM_VDO_HASH_ZONE_THREADS_MINIMUM	(0)
-#define DM_VDO_HASH_ZONE_THREADS_MAXIMUM	(100)
+#define DM_VDO_HASH_ZONE_THREADS_MINIMUM	0
+#define DM_VDO_HASH_ZONE_THREADS_MAXIMUM	100
 
-//#define DM_VDO_LOGICAL_THREADS_MINIMUM	(0)
-#define DM_VDO_LOGICAL_THREADS_MAXIMUM		(100)
+#define DM_VDO_LOGICAL_THREADS_MINIMUM		0
+#define DM_VDO_LOGICAL_THREADS_MAXIMUM		60
 
-//#define DM_VDO_PHYSICAL_THREADS_MINIMUM	(0)
-#define DM_VDO_PHYSICAL_THREADS_MAXIMUM		(16)
+#define DM_VDO_PHYSICAL_THREADS_MINIMUM		0
+#define DM_VDO_PHYSICAL_THREADS_MAXIMUM		16
 
-#define DM_VDO_MAX_DISCARD_MINIMUM		(1)
-#define DM_VDO_MAX_DISCARD_MAXIMUM		(UINT32_MAX / 4096)
+#define DM_VDO_MAX_DISCARD_MINIMUM		1
+#define DM_VDO_MAX_DISCARD_MAXIMUM		(UINT32_MAX / (uint32_t)(DM_VDO_BLOCK_SIZE_KB))
 
 #endif // DEVICE_MAPPER_VDO_LIMITS_H

device_mapper/vdo/vdo_reader.c

@@ -0,0 +1,279 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of the device-mapper userspace tools.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/*
+ * Based on VDO sources: https://github.com/dm-vdo/vdo
+ *
+ * Simplified parser of VDO superblock to obtain basic VDO parameteers
+ *
+ * TODO: maybe switch to some library in the future
+ */
+
+//#define _GNU_SOURCE 1
+//#define _LARGEFILE64_SOURCE 1
+
+#include "device_mapper/misc/dmlib.h"
+
+#include "target.h"
+
+#include "lib/mm/xlate.h"
+//#include "linux/byteorder/big_endian.h"
+//#include "linux/byteorder/little_endian.h"
+//#define le32_to_cpu __le32_to_cpu
+//#define le64_to_cpu __le64_to_cpu
+
+
+#include <errno.h>
+#include <fcntl.h>
+#include <linux/fs.h>	/* For block ioctl definitions */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+typedef unsigned char uuid_t[16];
+
+#define __packed	__attribute__((packed))
+
+static const char _MAGIC_NUMBER[] = "dmvdo001";
+#define MAGIC_NUMBER_SIZE  (sizeof(_MAGIC_NUMBER) - 1)
+
+struct vdo_version_number {
+	uint32_t major_version;
+	uint32_t minor_version;
+} __packed;
+
+/*
+ * The registry of component ids for use in headers
+ */
+enum {
+	SUPER_BLOCK	  = 0,
+	FIXED_LAYOUT	  = 1,
+	RECOVERY_JOURNAL  = 2,
+	SLAB_DEPOT	  = 3,
+	BLOCK_MAP	  = 4,
+	GEOMETRY_BLOCK	  = 5,
+}; /* ComponentID */
+
+struct vdo_header {
+	uint32_t id; /* The component this is a header for */
+	struct vdo_version_number version; /* The version of the data format */
+	size_t size; /* The size of the data following this header */
+} __packed;
+
+struct vdo_geometry_block {
+	char magic_number[MAGIC_NUMBER_SIZE];
+	struct vdo_header header;
+	uint32_t checksum;
+} __packed;
+
+struct vdo_config {
+	uint64_t logical_blocks; /* number of logical blocks */
+	uint64_t physical_blocks; /* number of physical blocks */
+	uint64_t slab_size; /* number of blocks in a slab */
+	uint64_t recovery_journal_size; /* number of recovery journal blocks */
+	uint64_t slab_journal_blocks; /* number of slab journal blocks */
+} __packed;
+
+struct vdo_component_41_0 {
+	uint32_t state;
+	uint64_t complete_recoveries;
+	uint64_t read_only_recoveries;
+	struct vdo_config config; /* packed */
+	uint64_t nonce;
+} __packed;
+
+enum vdo_volume_region_id {
+	VDO_INDEX_REGION = 0,
+	VDO_DATA_REGION = 1,
+	VDO_VOLUME_REGION_COUNT,
+};
+
+struct vdo_volume_region {
+	/* The ID of the region */
+	enum vdo_volume_region_id id;
+	/*
+	 * The absolute starting offset on the device. The region continues
+	 * until the next region begins.
+	 */
+	uint64_t start_block;
+} __packed;
+
+struct vdo_index_config {
+	uint32_t mem;
+	uint32_t unused;
+	uint8_t sparse;
+} __packed;
+
+struct vdo_volume_geometry {
+	uint32_t release_version;
+	uint64_t nonce;
+	uuid_t uuid;
+	uint64_t bio_offset;
+	struct vdo_volume_region regions[VDO_VOLUME_REGION_COUNT];
+	struct vdo_index_config index_config;
+} __packed;
+
+/* Decoding mostly only some used stucture members */
+
+static void _vdo_decode_version(struct vdo_version_number *v)
+{
+	v->major_version = le32_to_cpu(v->major_version);
+	v->minor_version = le32_to_cpu(v->minor_version);
+}
+
+static void _vdo_decode_header(struct vdo_header *h)
+{
+	h->id = le32_to_cpu(h->id);
+	_vdo_decode_version(&h->version);
+	h->size = le64_to_cpu(h->size);
+}
+
+static void _vdo_decode_geometry_region(struct vdo_volume_region *vr)
+{
+	vr->id = le32_to_cpu(vr->id);
+	vr->start_block = le32_to_cpu(vr->start_block);
+}
+
+static void _vdo_decode_volume_geometry(struct vdo_volume_geometry *vg)
+{
+	vg->release_version = le64_to_cpu(vg->release_version);
+	vg->nonce = le64_to_cpu(vg->nonce);
+	_vdo_decode_geometry_region(&vg->regions[VDO_DATA_REGION]);
+}
+
+static void _vdo_decode_config(struct vdo_config *vc)
+{
+	vc->logical_blocks = le64_to_cpu(vc->logical_blocks);
+	vc->physical_blocks = le64_to_cpu(vc->physical_blocks);
+	vc->slab_size = le64_to_cpu(vc->slab_size);
+	vc->recovery_journal_size = le64_to_cpu(vc->recovery_journal_size);
+	vc->slab_journal_blocks = le64_to_cpu(vc->slab_journal_blocks);
+}
+
+static void _vdo_decode_pvc(struct vdo_component_41_0 *pvc)
+{
+	_vdo_decode_config(&pvc->config);
+	pvc->nonce = le64_to_cpu(pvc->nonce);
+}
+
+bool dm_vdo_parse_logical_size(const char *vdo_path, uint64_t *logical_blocks)
+{
+	char buffer[4096];
+	int fh, n;
+	bool r = false;
+	off_t l;
+	struct stat st;
+	uint64_t size;
+	uint64_t regpos;
+
+	struct vdo_header h;
+	struct vdo_version_number vn;
+	struct vdo_volume_geometry vg;
+	struct vdo_component_41_0 pvc;
+
+	*logical_blocks = 0;
+	if ((fh = open(vdo_path, O_RDONLY)) == -1) {
+		log_sys_debug("Failed to open VDO backend %s.", vdo_path);
+		return false;
+	}
+
+	if (ioctl(fh, BLKGETSIZE64, &size) == -1) {
+		if (errno != ENOTTY) {
+			log_sys_debug("ioctl", vdo_path);
+			goto err;
+		}
+
+		/* lets retry for file sizes */
+		if (fstat(fh, &st) < 0) {
+			log_sys_debug("fstat", vdo_path);
+			goto err;
+		}
+
+		size = st.st_size;
+	}
+
+	if ((n = read(fh, buffer, sizeof(buffer))) < 0) {
+		log_sys_debug("read", vdo_path);
+		goto err;
+	}
+
+	if (strncmp(buffer, _MAGIC_NUMBER, MAGIC_NUMBER_SIZE)) {
+		log_debug_activation("Found mismatching VDO magic header in %s.", vdo_path);
+		goto err;
+	}
+
+	memcpy(&h, buffer + MAGIC_NUMBER_SIZE, sizeof(h));
+	_vdo_decode_header(&h);
+
+	if  (h.version.major_version != 5) {
+		log_debug_activation("Unsupported VDO version %u.%u.", h.version.major_version, h.version.minor_version);
+		goto err;
+	}
+
+	memcpy(&vg, buffer + MAGIC_NUMBER_SIZE + sizeof(h), sizeof(vg));
+	_vdo_decode_volume_geometry(&vg);
+
+	regpos = vg.regions[VDO_DATA_REGION].start_block * 4096;
+
+	if ((regpos + sizeof(buffer)) > size) {
+		log_debug_activation("File/Device is shorter and can't provide requested VDO volume region at " FMTu64 " > " FMTu64 ".", regpos, size);
+		goto err;
+	}
+
+	if ((l = lseek(fh, regpos, SEEK_SET)) < 0) {
+		log_sys_debug("lseek", vdo_path);
+		goto err;
+	}
+
+	if ((n = read(fh, buffer, sizeof(buffer))) < 0) {
+		log_sys_debug("read", vdo_path);
+		goto err;
+	}
+
+
+	memcpy(&vn, buffer + sizeof(struct vdo_geometry_block), sizeof(vn));
+	_vdo_decode_version(&vn);
+
+	if (vn.major_version > 41) {
+		log_debug_activation("Unknown VDO component version %u.", vn.major_version); // should be 41!
+		goto err;
+	}
+
+	memcpy(&pvc, buffer + sizeof(struct vdo_geometry_block) + sizeof(vn), sizeof(pvc));
+	_vdo_decode_pvc(&pvc);
+
+	if (pvc.nonce != vg.nonce) {
+		log_debug_activation("VDO metadata has mismatching VDO nonces " FMTu64 " != " FMTu64 ".", pvc.nonce, vg.nonce);
+		goto err;
+	}
+
+#if 0
+	log_debug_activation("LogBlocks    " FMTu64 ".", pvc.config.logical_blocks);
+	log_debug_activation("PhyBlocks    " FMTu64 ".", pvc.config.physical_blocks);
+	log_debug_activation("SlabSize     " FMTu64 ".", pvc.config.slab_size);
+	log_debug_activation("RecJourSize  " FMTu64 ".", pvc.config.recovery_journal_size);
+	log_debug_activation("SlabJouSize  " FMTu64 ".", pvc.config.slab_journal_blocks);
+#endif
+
+	*logical_blocks = pvc.config.logical_blocks;
+	r = true;
+err:
+	(void) close(fh);
+
+	return r;
+}

device_mapper/vdo/vdo_target.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2018-2022 Red Hat, Inc. All rights reserved.
  *
  * This file is part of LVM2.
  *
@@ -18,82 +18,117 @@
 #include "vdo_limits.h"
 #include "target.h"
 
+/* validate vdo target parameters and  'vdo_size' in sectors */
 bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 				   uint64_t vdo_size)
 {
 	bool valid = true;
 
 	/* 512 or 4096 bytes only ATM */
-	if ((vtp->minimum_io_size != 1) &&
-	    (vtp->minimum_io_size != 8)) {
-		log_error("VDO minimum io size %u is unsupported.",
+	if ((vtp->minimum_io_size != (512 >> SECTOR_SHIFT)) &&
+	    (vtp->minimum_io_size != (4096 >> SECTOR_SHIFT))) {
+		log_error("VDO minimum io size %u is unsupported [512, 4096].",
 			  vtp->minimum_io_size);
 		valid = false;
 	}
 
 	if ((vtp->block_map_cache_size_mb < DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB) ||
 	    (vtp->block_map_cache_size_mb > DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB)) {
-		log_error("VDO block map cache size %u out of range.",
-			  vtp->block_map_cache_size_mb);
+		log_error("VDO block map cache size %u MiB is out of range [%u..%u].",
+			  vtp->block_map_cache_size_mb,
+			  DM_VDO_BLOCK_MAP_CACHE_SIZE_MINIMUM_MB,
+			  DM_VDO_BLOCK_MAP_CACHE_SIZE_MAXIMUM_MB);
+		valid = false;
+	}
+
+	if ((vtp->block_map_era_length < DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM) ||
+	    (vtp->block_map_era_length > DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM)) {
+		log_error("VDO block map era length %u is out of range [%u..%u].",
+			  vtp->block_map_era_length,
+			  DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM,
+			  DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->index_memory_size_mb < DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB) ||
 	    (vtp->index_memory_size_mb > DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB)) {
-		log_error("VDO index memory size %u out of range.",
-			  vtp->index_memory_size_mb);
+		log_error("VDO index memory size %u MiB is out of range [%u..%u].",
+			  vtp->index_memory_size_mb,
+			  DM_VDO_INDEX_MEMORY_SIZE_MINIMUM_MB,
+			  DM_VDO_INDEX_MEMORY_SIZE_MAXIMUM_MB);
 		valid = false;
 	}
 
 	if ((vtp->slab_size_mb < DM_VDO_SLAB_SIZE_MINIMUM_MB) ||
 	    (vtp->slab_size_mb > DM_VDO_SLAB_SIZE_MAXIMUM_MB)) {
-		log_error("VDO slab size %u out of range.",
-			  vtp->slab_size_mb);
+		log_error("VDO slab size %u MiB is out of range [%u..%u].",
+			  vtp->slab_size_mb,
+			  DM_VDO_SLAB_SIZE_MINIMUM_MB,
+			  DM_VDO_SLAB_SIZE_MAXIMUM_MB);
 		valid = false;
 	}
 
 	if ((vtp->max_discard < DM_VDO_MAX_DISCARD_MINIMUM) ||
 	    (vtp->max_discard > DM_VDO_MAX_DISCARD_MAXIMUM)) {
-		log_error("VDO max discard %u out of range.",
-			  vtp->max_discard);
+		log_error("VDO max discard %u is out of range [%u..%u].",
+			  vtp->max_discard,
+			  DM_VDO_MAX_DISCARD_MINIMUM,
+			  DM_VDO_MAX_DISCARD_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->ack_threads > DM_VDO_ACK_THREADS_MAXIMUM) {
-		log_error("VDO ack threads %u out of range.", vtp->ack_threads);
+		log_error("VDO ack threads %u is out of range [0..%u].",
+			  vtp->ack_threads,
+			  DM_VDO_ACK_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->bio_threads < DM_VDO_BIO_THREADS_MINIMUM) ||
 	    (vtp->bio_threads > DM_VDO_BIO_THREADS_MAXIMUM)) {
-		log_error("VDO bio threads %u out of range.", vtp->bio_threads);
+		log_error("VDO bio threads %u is out of range [%u..%u].",
+			  vtp->bio_threads,
+			  DM_VDO_BIO_THREADS_MINIMUM,
+			  DM_VDO_BIO_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->bio_rotation < DM_VDO_BIO_ROTATION_MINIMUM) ||
 	    (vtp->bio_rotation > DM_VDO_BIO_ROTATION_MAXIMUM)) {
-		log_error("VDO bio rotation %u out of range.", vtp->bio_rotation);
+		log_error("VDO bio rotation %u is out of range [%u..%u].",
+			  vtp->bio_rotation,
+			  DM_VDO_BIO_ROTATION_MINIMUM,
+			  DM_VDO_BIO_ROTATION_MAXIMUM);
 		valid = false;
 	}
 
 	if ((vtp->cpu_threads < DM_VDO_CPU_THREADS_MINIMUM) ||
 	    (vtp->cpu_threads > DM_VDO_CPU_THREADS_MAXIMUM)) {
-		log_error("VDO cpu threads %u out of range.", vtp->cpu_threads);
+		log_error("VDO cpu threads %u is out of range [%u..%u].",
+			  vtp->cpu_threads,
+			  DM_VDO_CPU_THREADS_MINIMUM,
+			  DM_VDO_CPU_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->hash_zone_threads > DM_VDO_HASH_ZONE_THREADS_MAXIMUM) {
-		log_error("VDO hash zone threads %u out of range.", vtp->hash_zone_threads);
+		log_error("VDO hash zone threads %u is out of range [0..%u].",
+			  vtp->hash_zone_threads,
+			  DM_VDO_HASH_ZONE_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->logical_threads > DM_VDO_LOGICAL_THREADS_MAXIMUM) {
-		log_error("VDO logical threads %u out of range.", vtp->logical_threads);
+		log_error("VDO logical threads %u is out of range [0..%u].",
+			  vtp->logical_threads,
+			  DM_VDO_LOGICAL_THREADS_MAXIMUM);
 		valid = false;
 	}
 
 	if (vtp->physical_threads > DM_VDO_PHYSICAL_THREADS_MAXIMUM) {
-		log_error("VDO physical threads %u out of range.", vtp->physical_threads);
+		log_error("VDO physical threads %u is out of range [0..%u].",
+			  vtp->physical_threads,
+			  DM_VDO_PHYSICAL_THREADS_MAXIMUM);
 		valid = false;
 	}
 
@@ -120,10 +155,10 @@ bool dm_vdo_validate_target_params(const struct dm_vdo_target_params *vtp,
 		valid = false;
 	}
 
-	if (vdo_size >= (DM_VDO_LOGICAL_SIZE_MAXIMUM_MB * UINT64_C(1024 * 2))) {
-		log_error("VDO logical size is by " FMTu64 "KiB bigger then limit " FMTu64 "TiB.",
-			  (vdo_size - (DM_VDO_LOGICAL_SIZE_MAXIMUM_MB * UINT64_C(1024 * 2))) / 2,
-			  DM_VDO_LOGICAL_SIZE_MAXIMUM_MB / UINT64_C(1024) / UINT64_C(1024));
+	if (vdo_size > DM_VDO_LOGICAL_SIZE_MAXIMUM) {
+		log_error("VDO logical size is larger than limit " FMTu64 " TiB by " FMTu64 " KiB.",
+			  DM_VDO_LOGICAL_SIZE_MAXIMUM / (UINT64_C(1024) * 1024 * 1024 * 1024 >> SECTOR_SHIFT),
+			  (vdo_size - DM_VDO_LOGICAL_SIZE_MAXIMUM) / 2);
 		valid = false;
 	}
 

doc/kernel/cache-policies.txt

@@ -67,7 +67,7 @@ the entries (each hotspot block covers a larger area than a single
 cache block).
 
 All this means smq uses ~25bytes per cache block.  Still a lot of
-memory, but a substantial improvement nontheless.
+memory, but a substantial improvement nonetheless.
 
 Level balancing:
 mq placed entries in different levels of the multiqueue structures

doc/kernel/crypt.txt

@@ -35,7 +35,7 @@ Parameters: <cipher> <key> <iv_offset> <device path> \
         capi:authenc(hmac(sha256),xts(aes))-random
         capi:rfc7539(chacha20,poly1305)-random
 
-    The /proc/crypto contains a list of curently loaded crypto modes.
+    The /proc/crypto contains a list of currently loaded crypto modes.
 
 <key>
     Key used for encryption. It is encoded either as a hexadecimal number
@@ -81,7 +81,7 @@ Parameters: <cipher> <key> <iv_offset> <device path> \
 
 <#opt_params>
     Number of optional parameters. If there are no optional parameters,
-    the optional paramaters section can be skipped or #opt_params can be zero.
+    the optional parameters section can be skipped or #opt_params can be zero.
     Otherwise #opt_params is the number of following arguments.
 
     Example of optional parameters section:

doc/kernel/integrity.txt

@@ -120,7 +120,7 @@ journal_crypt:algorithm(:key)	(the key is optional)
 	"salsa20", "ctr(aes)" or "ecb(arc4)").
 
 	The journal contains history of last writes to the block device,
-	an attacker reading the journal could see the last sector nubmers
+	an attacker reading the journal could see the last sector numbers
 	that were written. From the sector numbers, the attacker can infer
 	the size of files that were written. To protect against this
 	situation, you can encrypt the journal.

doc/kernel/verity.txt

@@ -65,7 +65,7 @@ Construction Parameters
 
 <#opt_params>
     Number of optional parameters. If there are no optional parameters,
-    the optional paramaters section can be skipped or #opt_params can be zero.
+    the optional parameters section can be skipped or #opt_params can be zero.
     Otherwise #opt_params is the number of following arguments.
 
     Example of optional parameters section:

doc/lvm2-raid.txt

@@ -37,7 +37,7 @@ segment type.  The available RAID types are:
 	"raid6_nr" - RAID6 Rotating parity N with data restart
 	"raid6_nc" - RAID6 Rotating parity N with data continuation
 The exception to 'no shorthand options' will be where the RAID implementations
-can displace traditional tagets.  This is the case with 'mirror' and 'raid1'.
+can displace traditional targets.  This is the case with 'mirror' and 'raid1'.
 In this case, "mirror_segtype_default" - found under the "global" section in
 lvm.conf - can be set to "mirror" or "raid1".  The segment type inferred when
 the '-m' option is used will be taken from this setting.  The default segment
@@ -104,7 +104,7 @@ and 4 devices for RAID 6/10.
 
 lvconvert should work exactly as it does now when dealing with mirrors -
 even if(when) we switch to MD RAID1.  Of course, there are no plans to
-allow the presense of the metadata area to be configurable (e.g. --corelog).
+allow the presence of the metadata area to be configurable (e.g. --corelog).
 It will be simple enough to detect if the LV being up/down-converted is
 new or old-style mirroring.
 
@@ -120,7 +120,7 @@ RAID4 to RAID5 or RAID5 to RAID6.
 Line 02/03/04:
 These are familiar options - all of which would now be available as options
 for change.  (However, it'd be nice if we didn't have regionsize in there.
-It's simple on the kernel side, but is just an extra - often unecessary -
+It's simple on the kernel side, but is just an extra - often unnecessary -
 parameter to many functions in the LVM codebase.)
 
 Line 05:
@@ -375,8 +375,8 @@ the slot.  Even the names of the images will be renamed to properly reflect
 their index in the array.  Unlike the "mirror" segment type, you will never have
 an image named "*_rimage_1" occupying the index position 0.
 
-As with adding images, removing images holds off on commiting LVM metadata
-until all possible changes have been made.  This reduces the likelyhood of bad
+As with adding images, removing images holds off on committing LVM metadata
+until all possible changes have been made.  This reduces the likelihood of bad
 intermediate stages being left due to a failure of operation or machine crash.
 
 RAID1 '--splitmirrors', '--trackchanges', and '--merge' operations

doc/lvm_fault_handling.txt

@@ -87,7 +87,7 @@ are as follows:
   /etc/lvm/lvm.conf.  Once this operation is complete, the logical volumes
   will be consistent.  However, the volume group will still be inconsistent -
   due to the refernced-but-missing device/PV - and operations will still be
-  restricted to the aformentioned actions until either the device is
+  restricted to the aforementioned actions until either the device is
   restored or 'vgreduce --removemissing' is run.
 
 Device Revival (transient failures):
@@ -135,9 +135,9 @@ If a mirror is not 'in-sync', a read failure will produce an I/O error.
 This error will propagate all the way up to the applications above the
 logical volume (e.g. the file system).  No automatic intervention will
 take place in this case either.  It is up to the user to decide what
-can be done/salvaged in this senario.  If the user is confident that the
+can be done/salvaged in this scenario.  If the user is confident that the
 images of the mirror are the same (or they are willing to simply attempt
-to retreive whatever data they can), 'lvconvert' can be used to eliminate
+to retrieve whatever data they can), 'lvconvert' can be used to eliminate
 the failed image and proceed.
 
 Mirror resynchronization errors:
@@ -191,11 +191,11 @@ command are set in the LVM configuration file.  They are:
   3-way mirror fails, the mirror will be converted to a 2-way mirror.
   The "allocate" policy takes the further action of trying to replace
   the failed image using space that is available in the volume group.
-  Replacing a failed mirror image will incure the cost of
+  Replacing a failed mirror image will incur the cost of
   resynchronizing - degrading the performance of the mirror.  The
   default policy for handling an image failure is "remove".  This
   allows the mirror to still function, but gives the administrator the
-  choice of when to incure the extra performance costs of replacing
+  choice of when to incur the extra performance costs of replacing
   the failed image.
 
 RAID logical volume device failures are handled differently from the "mirror"

doc/lvmpolld_overview.txt

@@ -63,7 +63,7 @@ classical snapshot merge, thin snapshot merge.
 
 The second store is suited only for pvmove --abort operations in-progress. Both
 stores are independent and identical LVs (pvmove /dev/sda3 and pvmove --abort /dev/sda3)
-can be run concurently from lvmpolld point of view (on lvm2 side the consistency is
+can be run concurrently from lvmpolld point of view (on lvm2 side the consistency is
 guaranteed by lvm2 locking mechanism).
 
 Locking order

doc/tagging.txt

@@ -126,7 +126,7 @@ Usage Examples
   followed by 'vgchange -ay vg2'
   
   
-  Option (ii) - localised admin & configuation
+  Option (ii) - localised admin & configuration
   (i.e. each host holds *locally* which classes of volumes to activate)
     # Add @database tag to vg1's metadata
     vgchange --addtag @database vg1

doc/udev_assembly.txt

@@ -35,7 +35,7 @@ VGs from PVs as they appear, and at the same time collect information on what is
 already available. A command, pvscan --cache is expected to be used to
 implement udev rules. It is relatively easy to make this command print out a
 list of VGs (and possibly LVs) that have been made available by adding any
-particular device to the set of visible devices. In othe words, udev says "hey,
+particular device to the set of visible devices. In other words, udev says "hey,
 /dev/sdb just appeared", calls pvscan --cache, which talks to lvmetad, which
 says "cool, that makes vg0 complete". Pvscan takes this info and prints it out,
 and the udev rule can then somehow decide whether anything needs to be done

include/Makefile.in

@@ -18,6 +18,21 @@ top_builddir = @top_builddir@
 
 include $(top_builddir)/make.tmpl
 
+cmds.h:
+	@echo "    [GEN] $@"
+	$(Q) set -o pipefail && \
+	( cat $(top_srcdir)/tools/license.inc && \
+	  echo "/* Do not edit. This file is generated by the Makefile. */" && \
+	  echo "cmd(CMD_NONE, none)" && \
+	  $(GREP) '^ID:' $(top_srcdir)/tools/command-lines.in | LC_ALL=C $(SORT) -u | $(AWK) '{print "cmd(" $$2 "_CMD, " $$2 ")"}' && \
+	  echo "cmd(CMD_COUNT, count)" \
+	) > $@
+
+all: cmds.h
+
+clean:
+	rm -f cmds.h
+
 DISTCLEAN_TARGETS += configure.h lvm-version.h
 CLEAN_TARGETS += \
  .symlinks \
@@ -99,4 +114,5 @@ CLEAN_TARGETS += \
  util.h \
  uuid.h \
  vg.h \
- xlate.h
+ xlate.h \
+ cmds.h

include/configure.h.in

@@ -25,18 +25,13 @@
 /* The path to 'cache_restore', if available. */
 #undef CACHE_RESTORE_CMD
 
-/* Define to 1 if the `closedir' function returns void instead of `int'. */
+/* Define to 1 if the `closedir' function returns void instead of int. */
 #undef CLOSEDIR_VOID
 
 /* Path to cmirrord pidfile. */
 #undef CMIRRORD_PIDFILE
 
-/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
-   systems. This function is required for `alloca.c' support on those systems.
-   */
-#undef CRAY_STACKSEG_END
-
-/* Define to 1 if using `alloca.c'. */
+/* Define to 1 if using 'alloca.c'. */
 #undef C_ALLOCA
 
 /* Name of default metadata archive subdirectory. */
@@ -90,7 +85,7 @@
 /* Use blkid wiping by default. */
 #undef DEFAULT_USE_BLKID_WIPING
 
-/* Default for lvm.conf use_devicefile. */
+/* Default for lvm.conf use_devicesfile. */
 #undef DEFAULT_USE_DEVICES_FILE
 
 /* Use lvmlockd by default. */
@@ -141,11 +136,10 @@
 /* Define to 1 if you have the `alarm' function. */
 #undef HAVE_ALARM
 
-/* Define to 1 if you have `alloca', as a function or macro. */
+/* Define to 1 if you have 'alloca', as a function or macro. */
 #undef HAVE_ALLOCA
 
-/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix).
-   */
+/* Define to 1 if <alloca.h> works. */
 #undef HAVE_ALLOCA_H
 
 /* Define to 1 if you have the <arpa/inet.h> header file. */
@@ -273,6 +267,9 @@
 /* Define to 1 if you have the <machine/endian.h> header file. */
 #undef HAVE_MACHINE_ENDIAN_H
 
+/* Define to 1 if you have the `mallinfo2' function. */
+#undef HAVE_MALLINFO2
+
 /* Define to 1 if your system has a GNU libc compatible `malloc' function, and
    to 0 otherwise. */
 #undef HAVE_MALLOC
@@ -283,9 +280,6 @@
 /* Define to 1 if you have the `memchr' function. */
 #undef HAVE_MEMCHR
 
-/* Define to 1 if you have the <memory.h> header file. */
-#undef HAVE_MEMORY_H
-
 /* Define to 1 if you have the `memset' function. */
 #undef HAVE_MEMSET
 
@@ -402,7 +396,7 @@
 /* Define to 1 if you have the `strerror' function. */
 #undef HAVE_STRERROR
 
-/* Define to 1 if you have the `strerror_r' function. */
+/* Define if you have `strerror_r'. */
 #undef HAVE_STRERROR_R
 
 /* Define to 1 if you have the <strings.h> header file. */
@@ -604,6 +598,9 @@
 /* Path to lvm binary. */
 #undef LVM_PATH
 
+/* Path to lvresize_fs_helper script. */
+#undef LVRESIZE_FS_HELPER_PATH
+
 /* Define to 1 if `major', `minor', and `makedev' are declared in <mkdev.h>.
    */
 #undef MAJOR_IN_MKDEV
@@ -648,9 +645,6 @@
 /* Define to 1 to include the LVM readline shell. */
 #undef READLINE_SUPPORT
 
-/* Define as the return type of signal handlers (`int' or `void'). */
-#undef RETSIGTYPE
-
 /* Define to 1 to include built-in support for snapshots. */
 #undef SNAPSHOT_INTERNAL
 
@@ -662,7 +656,9 @@
 	STACK_DIRECTION = 0 => direction of growth unknown */
 #undef STACK_DIRECTION
 
-/* Define to 1 if you have the ANSI C header files. */
+/* Define to 1 if all of the C90 standard headers exist (not just the ones
+   required in a freestanding environment). This macro is provided for
+   backward compatibility; new code need not use it. */
 #undef STDC_HEADERS
 
 /* Define to 1 if strerror_r returns char *. */
@@ -693,9 +689,6 @@
 /* The path to 'thin_restore', if available. */
 #undef THIN_RESTORE_CMD
 
-/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
-#undef TIME_WITH_SYS_TIME
-
 /* Define to 1 if your <sys/time.h> declares `struct tm'. */
 #undef TM_IN_SYS_TIME
 
@@ -772,7 +765,7 @@
 /* Define to `long int' if <sys/types.h> does not define. */
 #undef off_t
 
-/* Define to `int' if <sys/types.h> does not define. */
+/* Define as a signed integer type capable of holding a process identifier. */
 #undef pid_t
 
 /* Define to rpl_realloc if the replacement function should be used. */

lib/Makefile.in

@@ -40,7 +40,10 @@ SOURCES =\
 	device/dev-luks.c \
 	device/dev-dasd.c \
 	device/dev-lvm1-pool.c \
+	device/filesystem.c \
 	device/online.c \
+	device/parse_vpd.c \
+	device/dev_util.c \
 	display/display.c \
 	error/errseg.c \
 	unknown/unknown.c \
@@ -54,7 +57,6 @@ SOURCES =\
 	filters/filter-partitioned.c \
 	filters/filter-type.c \
 	filters/filter-usable.c \
-	filters/filter-internal.c \
 	filters/filter-signature.c \
 	filters/filter-deviceid.c \
 	format_text/archive.c \

lib/activate/activate.c

@@ -322,6 +322,11 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent)
 {
 	return 0;
 }
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg)
+{
+	return 0;
+}
 int lvs_in_vg_activated(const struct volume_group *vg)
 {
 	return 0;
@@ -1363,6 +1368,32 @@ int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent)
 	return 1;
 }
 
+/*
+ * lv_vdo_pool_size_config obtains size configuration from active VDO table line
+ *
+ * If the 'params' string has been already retrieved, use it.
+ * If the mempool already exists, use it.
+ *
+ */
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg)
+{
+	struct dev_manager *dm;
+	int r;
+
+	if (!lv_info(lv->vg->cmd, lv, 1, NULL, 0, 0))
+		return 1;  /* Inactive VDO pool -> no runtime config */
+
+	if (!(dm = dev_manager_create(lv->vg->cmd, lv->vg->name, !lv_is_pvmove(lv))))
+		return_0;
+
+	r = dev_manager_vdo_pool_size_config(dm, lv, cfg);
+
+	dev_manager_destroy(dm);
+
+	return r;
+}
+
 static int _lv_active(struct cmd_context *cmd, const struct logical_volume *lv)
 {
 	struct lvinfo info;
@@ -2141,7 +2172,11 @@ static int _lv_suspend(struct cmd_context *cmd, const char *lvid_s,
 	 * TODO: Relax this limiting condition further */
 	if (!flush_required &&
 	    (lv_is_pvmove(lv) || pvmove_lv ||
-	     (!lv_is_mirror(lv) && !lv_is_thin_pool(lv) && !lv_is_thin_volume(lv)))) {
+	     (!lv_is_mirror(lv) &&
+	      !lv_is_thin_volume(lv) &&
+	      !lv_is_thin_pool(lv) &&
+	      !lv_is_vdo(lv) &&
+	      !lv_is_vdo_pool(lv)))) {
 		log_debug("Requiring flush for LV %s.", display_lvname(lv));
 		flush_required = 1;
 	}
@@ -2760,7 +2795,7 @@ static int _component_cb(struct logical_volume *lv, void *data)
 
 	if (lv_is_locked(lv) || lv_is_pvmove(lv) ||/* ignoring */
 	    /* thin-pool is special and it's using layered device */
-	    (lv_is_thin_pool(lv) && pool_is_active(lv)))
+	    (lv_is_thin_pool(lv) && thin_pool_is_active(lv)))
 		return -1;
 
 	/* External origin is activated through thinLV and uses -real suffix.

lib/activate/activate.h

@@ -204,6 +204,8 @@ int lv_thin_pool_status(const struct logical_volume *lv, int flush,
 int lv_vdo_pool_status(const struct logical_volume *lv, int flush,
 		       struct lv_status_vdo **status);
 int lv_vdo_pool_percent(const struct logical_volume *lv, dm_percent_t *percent);
+int lv_vdo_pool_size_config(const struct logical_volume *lv,
+			    struct vdo_pool_size_config *cfg);
 
 /*
  * Return number of LVs in the VG that are active.

lib/activate/dev_manager.c

@@ -277,7 +277,7 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 	int dmtask;
 	int with_flush; /* TODO: arg for _info_run */
 	void *target = NULL;
-	uint64_t target_start, target_length, start, length, length_crop = 0;
+	uint64_t target_start, target_length, start, extent_size, length, length_crop = 0;
 	char *target_name, *target_params;
 	const char *devname;
 
@@ -306,8 +306,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 
 	/* Query status only for active device */
 	if (seg_status && dminfo->exists) {
-		start = length = seg_status->seg->lv->vg->extent_size;
-		start *= seg_status->seg->le;
+		extent_size = length = seg_status->seg->lv->vg->extent_size;
+		start = extent_size * seg_status->seg->le;
 		length *= _seg_len(seg_status->seg);
 
 		/* Uses max DM_THIN_MAX_METADATA_SIZE sectors for metadata device */
@@ -328,6 +328,8 @@ static int _info_run(const char *dlid, struct dm_info *dminfo,
 
 			if ((start == target_start) &&
 			    ((length == target_length) ||
+			     ((lv_is_vdo_pool(seg_status->seg->lv)) && /* should fit within extent size */
+			      (length < target_length) && ((length + extent_size) > target_length)) ||
 			     (length_crop && (length_crop == target_length))))
 				break; /* Keep target_params when matching segment is found */
 
@@ -1955,6 +1957,71 @@ out:
 	return r;
 }
 
+int dev_manager_vdo_pool_size_config(struct dev_manager *dm,
+				     const struct logical_volume *lv,
+				     struct vdo_pool_size_config *cfg)
+{
+	const char *dlid;
+	struct dm_info info;
+	uint64_t start, length;
+	struct dm_task *dmt = NULL;
+	char *type = NULL;
+	char *params = NULL;
+	int r = 0;
+	unsigned version = 0;
+
+	memset(cfg, 0, sizeof(*cfg));
+
+	if (!(dlid = build_dm_uuid(dm->mem, lv, lv_layer(lv))))
+		return_0;
+
+	if (!(dmt = _setup_task_run(DM_DEVICE_TABLE, &info, NULL, dlid, 0, 0, 0, 0, 0, 0)))
+		return_0;
+
+	if (!info.exists)
+		goto inactive; /* VDO device is not active, should not happen here... */
+
+	log_debug_activation("Checking VDO pool table line for LV %s.",
+			     display_lvname(lv));
+
+	if (dm_get_next_target(dmt, NULL, &start, &length, &type, &params)) {
+		log_error("More then one table line found for %s.",
+			  display_lvname(lv));
+		goto out;
+	}
+
+	if (!type || strcmp(type, TARGET_NAME_VDO)) {
+		log_error("Expected %s segment type but got %s instead.",
+			  TARGET_NAME_VDO, type ? type : "NULL");
+		goto out;
+	}
+
+	if (sscanf(params, "V%u %*s " FMTu64 " %*u " FMTu32,
+		   &version, &cfg->physical_size, &cfg->block_map_cache_size_mb) != 3) {
+		log_error("Failed to parse VDO parameters %s for LV %s.",
+			  params, display_lvname(lv));
+		goto out;
+	}
+
+	switch (version) {
+	case 2: break;
+	case 4: break;
+	default: log_warn("WARNING: Unknown VDO table line version %u.", version);
+	}
+
+	cfg->virtual_size = length;
+	cfg->physical_size *= 8; // From 4K unit to 512B
+	cfg->block_map_cache_size_mb /= 256; // From 4K unit to MiB
+	cfg->index_memory_size_mb = first_seg(lv)->vdo_params.index_memory_size_mb; // Preserved
+
+inactive:
+	r = 1;
+out:
+	dm_task_destroy(dmt);
+
+	return r;
+}
+
 
 /*************************/
 /*  NEW CODE STARTS HERE */
@@ -3842,11 +3909,12 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv,
 	if (!seg_is_striped_target(first_seg(lv)) || (action == CLEAN))
 		dm->cmd->disable_dm_devs = 1;
 
-	if (!(dtree = _create_partial_dtree(dm, lv, laopts->origin_only)))
-		return_0;
-
+	dtree = _create_partial_dtree(dm, lv, laopts->origin_only);
 	dm->cmd->disable_dm_devs = tmp_state;
 
+	if (!dtree)
+		return_0;
+
 	if (!(root = dm_tree_find_node(dtree, 0, 0))) {
 		log_error("Lost dependency tree root node.");
 		goto out_no_root;
@@ -3907,6 +3975,8 @@ static int _tree_action(struct dev_manager *dm, const struct logical_volume *lv,
 		 * non 'thin pool/volume' and  size increase */
 		else if (!lv_is_thin_volume(lv) &&
 			 !lv_is_thin_pool(lv) &&
+			 !lv_is_vdo(lv) &&
+			 !lv_is_vdo_pool(lv) &&
 			 dm_tree_node_size_changed(root))
 			dm->flush_required = 1;
 
@@ -4025,3 +4095,78 @@ out:
 
 	return r;
 }
+
+/*
+ * crypt offset is usually the LUKS header size but can be larger.
+ * The LUKS header is usually 2MB for LUKS1 and 16MB for LUKS2.
+ * The offset needs to be subtracted from the LV size to get the
+ * size used to resize the crypt device.
+ */
+int get_crypt_table_offset(dev_t crypt_devt, uint32_t *offset_bytes)
+{
+	struct dm_task *dmt = dm_task_create(DM_DEVICE_TABLE);
+	uint64_t start, length;
+	char *target_type = NULL;
+	void *next = NULL;
+	char *params = NULL;
+	char offset_str[32] = { 0 };
+	int copy_offset = 0;
+	int spaces = 0;
+	int i, i_off = 0;
+
+	if (!dmt)
+		return_0;
+
+	if (!dm_task_set_major_minor(dmt, (int)MAJOR(crypt_devt), (int)MINOR(crypt_devt), 0)) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	/* Non-blocking status read */
+	if (!dm_task_no_flush(dmt))
+		log_warn("WARNING: Can't set no_flush for dm status.");
+
+	if (!dm_task_run(dmt)) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	next = dm_get_next_target(dmt, next, &start, &length, &target_type, &params);
+
+	if (!target_type || !params || strcmp(target_type, "crypt")) {
+		dm_task_destroy(dmt);
+		return_0;
+	}
+
+	/*
+	 * get offset from params string:
+	 * <cipher> <key> <iv_offset> <device> <offset> [<#opt_params> <opt_params>]
+	 * <offset> is reported in 512 byte sectors.
+	 */
+	for (i = 0; i < strlen(params); i++) {
+		if (params[i] == ' ') {
+			spaces++;
+			if (spaces == 4)
+				copy_offset = 1;
+			if (spaces == 5)
+				break;
+			continue;
+		}
+		if (!copy_offset)
+			continue;
+
+		offset_str[i_off++] = params[i];
+
+		if (i_off == sizeof(offset_str)) {
+			offset_str[0] = '\0';
+			break;
+		}
+	}
+	dm_task_destroy(dmt);
+
+	if (!offset_str[0])
+		return_0;
+
+	*offset_bytes = ((uint32_t)strtoul(offset_str, NULL, 0) * 512);
+	return 1;
+}

lib/activate/dev_manager.h

@@ -29,6 +29,8 @@ struct lv_seg_status;
 
 int read_only_lv(const struct logical_volume *lv, const struct lv_activate_opts *laopts, const char *layer);
 
+int get_crypt_table_offset(dev_t crypt_devt, uint32_t *offset_bytes);
+
 /*
  * Constructor and destructor.
  */
@@ -81,6 +83,9 @@ int dev_manager_thin_pool_status(struct dev_manager *dm,
 int dev_manager_vdo_pool_status(struct dev_manager *dm,
 				const struct logical_volume *lv, int flush,
 				struct lv_status_vdo **status, int *exists);
+int dev_manager_vdo_pool_size_config(struct dev_manager *dm,
+				     const struct logical_volume *lv,
+				     struct vdo_pool_size_config *cfg);
 int dev_manager_suspend(struct dev_manager *dm, const struct logical_volume *lv,
 			struct lv_activate_opts *laopts, int lockfs, int flush_required);
 int dev_manager_activate(struct dev_manager *dm, const struct logical_volume *lv,

lib/cache/lvmcache.c

@@ -144,28 +144,6 @@ int lvmcache_found_duplicate_vgnames(void)
 	return _found_duplicate_vgnames;
 }
 
-static struct device_list *_get_devl_in_device_list(struct device *dev, struct dm_list *head)
-{
-	struct device_list *devl;
-
-	dm_list_iterate_items(devl, head) {
-		if (devl->dev == dev)
-			return devl;
-	}
-	return NULL;
-}
-
-int dev_in_device_list(struct device *dev, struct dm_list *head)
-{
-	struct device_list *devl;
-
-	dm_list_iterate_items(devl, head) {
-		if (devl->dev == dev)
-			return 1;
-	}
-	return 0;
-}
-
 bool lvmcache_has_duplicate_devs(void)
 {
 	if (dm_list_empty(&_unused_duplicates) && dm_list_empty(&_initial_duplicates))
@@ -192,11 +170,11 @@ void lvmcache_del_dev_from_duplicates(struct device *dev)
 {
 	struct device_list *devl;
 
-	if ((devl = _get_devl_in_device_list(dev, &_initial_duplicates))) {
+	if ((devl = device_list_find_dev(&_initial_duplicates, dev))) {
 		log_debug_cache("delete dev from initial duplicates %s", dev_name(dev));
 		dm_list_del(&devl->list);
 	}
-	if ((devl = _get_devl_in_device_list(dev, &_unused_duplicates))) {
+	if ((devl = device_list_find_dev(&_unused_duplicates, dev))) {
 		log_debug_cache("delete dev from unused duplicates %s", dev_name(dev));
 		dm_list_del(&devl->list);
 	}
@@ -605,7 +583,7 @@ int vg_has_duplicate_pvs(struct volume_group *vg)
 
 bool lvmcache_dev_is_unused_duplicate(struct device *dev)
 {
-	return dev_in_device_list(dev, &_unused_duplicates) ? true : false;
+	return device_list_find_dev(&_unused_duplicates, dev) ? true : false;
 }
 
 static void _warn_unused_duplicates(struct cmd_context *cmd)
@@ -644,14 +622,19 @@ static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_in
 	struct device *dev_mp = NULL;
 	struct device *dev1 = NULL;
 	struct device *dev;
+	char wwid1_buf[DEV_WWID_SIZE] = { 0 };
+	char wwid_buf[DEV_WWID_SIZE] = { 0 };
 	const char *wwid1 = NULL;
-	const char *wwid;
+	const char *wwid = NULL;
 	int diff_wwid = 0;
 	int same_wwid = 0;
 	int dev_is_mp;
 
 	*dev_mpath = NULL;
 
+	if (!find_config_tree_bool(cmd, devices_multipath_component_detection_CFG, NULL))
+		return 0;
+
 	/* This function only makes sense with more than one dev. */
 	if ((info && dm_list_empty(altdevs)) || (!info && (dm_list_size(altdevs) == 1))) {
 		log_debug("Skip multipath component checks with single device for PVID %s", pvid);
@@ -664,14 +647,23 @@ static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_in
 		dev = info->dev;
 		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
 
+		/*
+		 * dev_mpath_component_wwid allocates wwid from dm_pool,
+		 * device_id_system_read does not and needs free.
+		 */
+
 		if (dev_is_mp) {
 			if ((wwid1 = dev_mpath_component_wwid(cmd, dev))) {
+				strncpy(wwid1_buf, wwid1, DEV_WWID_SIZE-1);
 				dev_mp = dev;
 				dev1 = dev;
 			}
 		} else {
-			if ((wwid1 = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID)))
+			if ((wwid1 = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID))) {
+				strncpy(wwid1_buf, wwid1, DEV_WWID_SIZE-1);
+				free((char *)wwid1);
 				dev1 = dev;
+			}
 		}
 	}
 
@@ -679,31 +671,36 @@ static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_in
 		dev = devl->dev;
 		dev_is_mp = (cmd->dev_types->device_mapper_major == MAJOR(dev->dev)) && dev_has_mpath_uuid(cmd, dev, NULL);
 
-		if (dev_is_mp)
-			wwid = dev_mpath_component_wwid(cmd, dev);
-		else
-			wwid = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID);
+		if (dev_is_mp) {
+			if ((wwid = dev_mpath_component_wwid(cmd, dev)))
+				strncpy(wwid_buf, wwid, DEV_WWID_SIZE-1);
+		} else {
+			if ((wwid = device_id_system_read(cmd, dev, DEV_ID_TYPE_SYS_WWID))) {
+				strncpy(wwid_buf, wwid, DEV_WWID_SIZE-1);
+				free((char *)wwid);
+			}
+		}
 
-		if (!wwid && wwid1) {
+		if (!wwid_buf[0] && wwid1_buf[0]) {
 			log_debug("Different wwids for duplicate PVs %s %s %s none",
-				  dev_name(dev1), wwid1, dev_name(dev));
+				  dev_name(dev1), wwid1_buf, dev_name(dev));
 			diff_wwid++;
 			continue;
 		}
 
-		if (!wwid)
+		if (!wwid_buf[0])
 			continue;
 
-		if (!wwid1) {
-			wwid1 = wwid;
+		if (!wwid1_buf[0]) {
+			memcpy(wwid1_buf, wwid_buf, DEV_WWID_SIZE-1);
 			dev1 = dev;
 			continue;
 		}
 
 		/* Different wwids indicates these are not multipath components. */
-		if (strcmp(wwid1, wwid)) {
+		if (strcmp(wwid1_buf, wwid_buf)) {
 			log_debug("Different wwids for duplicate PVs %s %s %s %s",
-				  dev_name(dev1), wwid1, dev_name(dev), wwid);
+				  dev_name(dev1), wwid1_buf, dev_name(dev), wwid_buf);
 			diff_wwid++;
 			continue;
 		}
@@ -711,7 +708,7 @@ static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_in
 		/* Different mpath devs with the same wwid shouldn't happen. */
 		if (dev_is_mp && dev_mp) {
 			log_print("Found multiple multipath devices for PVID %s WWID %s: %s %s",
-				   pvid, wwid1, dev_name(dev_mp), dev_name(dev));
+				   pvid, wwid1_buf, dev_name(dev_mp), dev_name(dev));
 			continue;
 		}
 
@@ -727,7 +724,7 @@ static int _all_multipath_components(struct cmd_context *cmd, struct lvmcache_in
 		return 0;
 
 	if (dev_mp)
-		log_debug("Found multipath device %s for PVID %s WWID %s.", dev_name(dev_mp), pvid, wwid1);
+		log_debug("Found multipath device %s for PVID %s WWID %s.", dev_name(dev_mp), pvid, wwid1_buf);
 
 	*dev_mpath = dev_mp;
 	return 1;
@@ -915,7 +912,7 @@ next:
 			}
 
 			/* Remove dev_mpath from altdevs. */
-			if ((devl = _get_devl_in_device_list(dev_mpath, &altdevs)))
+			if ((devl = device_list_find_dev(&altdevs, dev_mpath)))
 				dm_list_del(&devl->list);
 
 			/* Remove info from lvmcache that came from the component dev. */
@@ -982,7 +979,7 @@ next:
 			}
 
 			/* Remove dev_md from altdevs. */
-			if ((devl = _get_devl_in_device_list(dev_md, &altdevs)))
+			if ((devl = device_list_find_dev(&altdevs, dev_md)))
 				dm_list_del(&devl->list);
 
 			/* Remove info from lvmcache that came from the component dev. */
@@ -1010,7 +1007,7 @@ next:
 			}
 
 			/* Remove dev_md from altdevs. */
-			if ((devl = _get_devl_in_device_list(dev_md, &altdevs)))
+			if ((devl = device_list_find_dev(&altdevs, dev_md)))
 				dm_list_del(&devl->list);
 		}
 
@@ -1088,8 +1085,8 @@ next:
 		if (dev1 == dev2)
 			continue;
 
-		prev_unchosen1 = dev_in_device_list(dev1, &_unused_duplicates);
-		prev_unchosen2 = dev_in_device_list(dev2, &_unused_duplicates);
+		prev_unchosen1 = device_list_find_dev(&_unused_duplicates, dev1) ? 1 :0;
+		prev_unchosen2 = device_list_find_dev(&_unused_duplicates, dev2) ? 1 :0;
 
 		if (!prev_unchosen1 && !prev_unchosen2) {
 			/*
@@ -1099,8 +1096,8 @@ next:
 			 * want the same duplicate preference to be preserved
 			 * in each instance of lvmcache for a single command.
 			 */
-			prev_unchosen1 = dev_in_device_list(dev1, &_prev_unused_duplicate_devs);
-			prev_unchosen2 = dev_in_device_list(dev2, &_prev_unused_duplicate_devs);
+			prev_unchosen1 = device_list_find_dev(&_prev_unused_duplicate_devs, dev1) ? 1 :0;
+			prev_unchosen2 = device_list_find_dev(&_prev_unused_duplicate_devs, dev2) ? 1 : 0;
 		}
 
 		dev1_major = MAJOR(dev1->dev);
@@ -1277,7 +1274,7 @@ next:
 	if (!info) {
 		log_debug_cache("PV %s with duplicates will use %s.", pvid, dev_name(dev1));
 
-		if (!(devl_add = _get_devl_in_device_list(dev1, &altdevs))) {
+		if (!(devl_add = device_list_find_dev(&altdevs, dev1))) {
 			/* shouldn't happen */
 			log_error(INTERNAL_ERROR "PV %s with duplicates no alternate list entry for %s", pvid, dev_name(dev1));
 			dm_list_splice(&new_unused, &altdevs);
@@ -1296,7 +1293,7 @@ next:
 		 * for the current lvmcache device to drop.
 		 */
 
-		if (!(devl_add = _get_devl_in_device_list(dev1, &altdevs))) {
+		if (!(devl_add = device_list_find_dev(&altdevs, dev1))) {
 			/* shouldn't happen */
 			log_error(INTERNAL_ERROR "PV %s with duplicates no alternate list entry for %s", pvid, dev_name(dev1));
 			dm_list_splice(&new_unused, &altdevs);
@@ -1612,7 +1609,24 @@ int lvmcache_label_scan(struct cmd_context *cmd)
 	 * with infos/vginfos based on reading headers from
 	 * each device, and a vg summary from each mda.
 	 */
-	label_scan(cmd);
+	if (!label_scan(cmd))
+		return_0;
+
+	/*
+	 * device_ids_validate() found devices using a sys_serial device id
+	 * which had a PVID on disk that did not match the PVID in the devices
+	 * file.  Serial numbers may not always be unique, so any device with
+	 * the same serial number is found and searched for the correct PVID.
+	 * If the PVID is found on a device that has not been scanned, then
+	 * it needs to be scanned so it can be used.
+	 */
+	if (!dm_list_empty(&cmd->device_ids_check_serial)) {
+		struct dm_list scan_devs;
+		dm_list_init(&scan_devs);
+		device_ids_check_serial(cmd, &scan_devs, NULL, 0);
+		if (!dm_list_empty(&scan_devs))
+			label_scan_devs(cmd, cmd->filter, &scan_devs);
+	}
 
 	/*
 	 * When devnames are used as device ids (which is dispreferred),
@@ -2494,7 +2508,7 @@ struct lvmcache_info *lvmcache_add(struct cmd_context *cmd, struct labeller *lab
 			memcpy(dev->pvid, pvid, ID_LEN);
 
 			/* shouldn't happen */
-			if (dev_in_device_list(dev, &_initial_duplicates))
+			if (device_list_find_dev(&_initial_duplicates, dev))
 				log_debug_cache("Initial duplicate already in list %s", dev_name(dev));
 			else {
 				/*

lib/cache/lvmcache.h

@@ -186,8 +186,6 @@ int lvmcache_vginfo_has_pvid(struct lvmcache_vginfo *vginfo, const char *pvid_ar
 uint64_t lvmcache_max_metadata_size(void);
 void lvmcache_save_metadata_size(uint64_t val);
 
-int dev_in_device_list(struct device *dev, struct dm_list *head);
-
 bool lvmcache_has_bad_metadata(struct device *dev);
 
 bool lvmcache_has_old_metadata(struct cmd_context *cmd, const char *vgname, const char *vgid, struct device *dev);

lib/commands/cmd_enum.h

@@ -0,0 +1,19 @@
+#ifndef _CMD_ENUM_H
+#define _CMD_ENUM_H
+
+/*
+ * include/cmds.h is generated by the Makefile.  For each command definition
+ * in command-lines.in, cmds.h contains:
+ * cmd(foo_CMD, foo)
+ *
+ * This header adds each of the foo_CMD's into an enum, so there's
+ * a unique integer identifier for each command definition.
+ */
+
+enum {
+#define cmd(a, b) a ,
+#include "../../include/cmds.h"
+#undef cmd
+};
+
+#endif

lib/commands/toolcontext.c

@@ -643,6 +643,7 @@ static int _process_config(struct cmd_context *cmd)
 	if (!dm_set_uuid_prefix(UUID_PREFIX))
 		return_0;
 #endif
+	cmd->device_id_sysfs_dir = find_config_tree_str(cmd, devices_device_id_sysfs_dir_CFG, NULL);
 
 	dev_ext_info_src = find_config_tree_str(cmd, devices_external_device_info_source_CFG, NULL);
 
@@ -1128,7 +1129,7 @@ static int _init_dev_cache(struct cmd_context *cmd)
 	return 1;
 }
 
-#define MAX_FILTERS 11
+#define MAX_FILTERS 10
 
 static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 {
@@ -1143,13 +1144,6 @@ static struct dev_filter *_init_filter_chain(struct cmd_context *cmd)
 	 * Update MAX_FILTERS definition above when adding new filters.
 	 */
 
-	/* internal filter used by command processing. */
-	if (!(filters[nr_filt] = internal_filter_create())) {
-		log_error("Failed to create internal device filter");
-		goto bad;
-	}
-	nr_filt++;
-
 	/* global regex filter. Optional. */
 	if ((cn = find_config_tree_node(cmd, devices_global_filter_CFG, NULL))) {
 		if (!(filters[nr_filt] = regex_filter_create(cn->v, 0, 1))) {
@@ -1912,7 +1906,6 @@ int refresh_toolcontext(struct cmd_context *cmd)
 	_destroy_segtypes(&cmd->segtypes);
 	_destroy_formats(cmd, &cmd->formats);
 
-	devices_file_exit(cmd);
 	if (!dev_cache_exit())
 		stack;
 	_destroy_dev_types(cmd);
@@ -2041,7 +2034,6 @@ void destroy_toolcontext(struct cmd_context *cmd)
 	_destroy_segtypes(&cmd->segtypes);
 	_destroy_formats(cmd, &cmd->formats);
 	_destroy_filters(cmd);
-	devices_file_exit(cmd);
 	dev_cache_exit();
 	_destroy_dev_types(cmd);
 	_destroy_tags(cmd);

lib/commands/toolcontext.h

@@ -18,6 +18,7 @@
 
 #include "lib/device/dev-cache.h"
 #include "lib/device/dev-type.h"
+#include "lib/commands/cmd_enum.h"
 
 #include <limits.h>
 
@@ -94,6 +95,7 @@ struct cmd_context {
 	const char *name; /* needed before cmd->command is set */
 	struct command_name *cname;
 	struct command *command;
+	int command_enum; /* duplicate from command->command_enum for lib code */
 	char **argv;
 	struct arg_values *opt_arg_values;
 	struct dm_list arg_value_groups;
@@ -144,6 +146,7 @@ struct cmd_context {
 	unsigned degraded_activation:1;
 	unsigned auto_set_activation_skip:1;
 	unsigned si_unit_consistency:1;
+	unsigned report_strict_type_mode:1;
 	unsigned report_binary_values_as_numeric:1;
 	unsigned report_mark_hidden_devices:1;
 	unsigned metadata_read_only:1;
@@ -212,6 +215,7 @@ struct cmd_context {
 	struct dm_list use_devices;		/* struct dev_use for each entry in devices file */
 	const char *md_component_checks;
 	const char *search_for_devnames;	/* config file setting */
+	struct dm_list device_ids_check_serial;
 	const char *devicesfile;                /* from --devicesfile option */
 	struct dm_list deviceslist;             /* from --devices option, struct dm_str_list */
 
@@ -244,6 +248,7 @@ struct cmd_context {
 	 * Paths.
 	 */
 	const char *lib_dir;			/* cache value global/library_dir */
+	const char *device_id_sysfs_dir;
 	char system_dir[PATH_MAX];
 	char dev_dir[PATH_MAX];
 	char proc_dir[PATH_MAX];

lib/config/config.c

@@ -1830,8 +1830,9 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 	char summary[MAX_COMMENT_LINE+1];
 	char version[9];
 	int pos = 0;
+	int space_prefix_len = 0;
+	const char *p;
 	size_t len;
-	char *space_prefix;
 
 	if ((out->tree_spec->type == CFG_DEF_TREE_DIFF) &&
 	    (!(out->tree_spec->check_status[cn->id] & CFG_DIFF)))
@@ -1865,16 +1866,36 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 
 	/* Usual tree view with nodes and their values. */
 
+	if (out->tree_spec->valuesonly && !(cfg_def->type & CFG_TYPE_SECTION)) {
+		if ((space_prefix_len = strspn(line, "\t "))) {
+			len = strlen(line);
+			p = line + space_prefix_len;
+
+			/* copy space_prefix, skip key and '=', copy value */
+			if (!dm_pool_begin_object(out->mem, len))
+				return_0;
+
+			if (!dm_pool_grow_object(out->mem, line, space_prefix_len) ||
+			    !dm_pool_grow_object(out->mem, p + strcspn(p, "=") + 1, len + 1)) {
+				dm_pool_abandon_object(out->mem);
+				return_0;
+			}
+
+			line = dm_pool_end_object(out->mem);
+		} else
+			line = strchr(line, '=') + 1;
+	}
+
 	if ((out->tree_spec->type != CFG_DEF_TREE_CURRENT) &&
 	    (out->tree_spec->type != CFG_DEF_TREE_DIFF) &&
 	    (out->tree_spec->type != CFG_DEF_TREE_FULL) &&
+	    !out->tree_spec->valuesonly &&
 	    (cfg_def->flags & (CFG_DEFAULT_UNDEFINED | CFG_DEFAULT_COMMENTED))) {
 		/* print with # at the front to comment out the line */
 		if (_should_print_cfg_with_undef_def_val(out, cfg_def, cn)) {
-			space_prefix = ((len = strspn(line, "\t "))) ? dm_pool_strndup(out->mem, line, len) : NULL;
-			fprintf(out->fp, "%s%s%s\n", space_prefix ? : "", "# ", line + len);
-			if (space_prefix)
-				dm_pool_free(out->mem, space_prefix);
+			space_prefix_len = strspn(line, "\t ");
+			fprintf(out->fp, "%.*s%s%s\n", space_prefix_len, line, "# ",
+				line + space_prefix_len);
 		}
 		return 1;
 	}
@@ -1883,6 +1904,9 @@ static int _out_line_fn(const struct dm_config_node *cn, const char *line, void
 	if (_should_print_cfg_with_undef_def_val(out, cfg_def, cn))
 		fprintf(out->fp, "%s\n", line);
 
+	if (out->tree_spec->valuesonly && !(cfg_def->type & CFG_TYPE_SECTION) && space_prefix_len)
+		dm_pool_free(out->mem, (char *) line);
+
 	return 1;
 }
 

lib/config/config.h

@@ -176,6 +176,7 @@ struct config_def_tree_spec {
 	unsigned unconfigured:1;		/* use unconfigured path strings */
 	unsigned withgeneralpreamble:1;		/* include preamble for a general config file */
 	unsigned withlocalpreamble:1;		/* include preamble for a local config file */
+	unsigned valuesonly:1;                  /* print only values without keys */
 	uint8_t *check_status;			/* status of last tree check (currently needed for CFG_DEF_TREE_MISSING only) */
 };
 

lib/config/config_settings.h

@@ -118,6 +118,7 @@
  * the previous default value was set (uncommented) in lvm.conf.
  */
 #include "lib/config/defaults.h"
+#include "device_mapper/vdo/vdo_limits.h"
 
 cfg_section(root_CFG_SECTION, "(root)", root_CFG_SECTION, 0, vsn(0, 0, 0), 0, NULL, NULL)
 
@@ -223,6 +224,9 @@ cfg(devices_dir_CFG, "dir", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_ADV
 	"Directory in which to create volume group device nodes.\n"
 	"Commands also accept this as a prefix on volume group names.\n")
 
+cfg(devices_device_id_sysfs_dir_CFG, "device_id_sysfs_dir", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_UNSUPPORTED, CFG_TYPE_STRING, DEFAULT_DEVICE_ID_SYSFS_DIR, vsn(2, 3, 17), NULL, 0, NULL,
+	"Location of sysfs for finding device ids (for testing.)\n")
+
 cfg_array(devices_scan_CFG, "scan", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED | CFG_ADVANCED, CFG_TYPE_STRING, "#S/dev", vsn(1, 0, 0), NULL, 0, NULL,
 	"Directories containing device nodes to use with LVM.\n")
 
@@ -276,7 +280,7 @@ cfg_array(devices_preferred_names_CFG, "preferred_names", devices_CFG_SECTION, C
 	"preferred_names = [ \"^/dev/mpath/\", \"^/dev/mapper/mpath\", \"^/dev/[hs]d\" ]\n"
 	"#\n")
 
-cfg(devices_use_devicesfile_CFG, "use_devicesfile", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_USE_DEVICES_FILE, vsn(2, 3, 12), NULL, 0, NULL,
+cfg(devices_use_devicesfile_CFG, "use_devicesfile", devices_CFG_SECTION, CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_USE_DEVICES_FILE, vsn(2, 3, 12), "@DEFAULT_USE_DEVICES_FILE@", 0, NULL,
 	"Enable or disable the use of a devices file.\n"
 	"When enabled, lvm will only use devices that\n"
 	"are lised in the devices file. A devices file will\n"
@@ -708,12 +712,11 @@ cfg(allocation_vdo_use_deduplication_CFG, "vdo_use_deduplication", allocation_CF
 cfg(allocation_vdo_use_metadata_hints_CFG, "vdo_use_metadata_hints", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_USE_METADATA_HINTS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Enables or disables whether VDO volume should tag its latency-critical\n"
 	"writes with the REQ_SYNC flag. Some device mapper targets such as dm-raid5\n"
-	"process writes with this flag at a higher priority.\n"
-	"Default is enabled.\n")
+	"process writes with this flag at a higher priority.\n")
 
 cfg(allocation_vdo_minimum_io_size_CFG, "vdo_minimum_io_size", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_MINIMUM_IO_SIZE, VDO_1ST_VSN, NULL, 0, NULL,
 	"The minimum IO size for VDO volume to accept, in bytes.\n"
-	"Valid values are 512 or 4096. The recommended and default value is 4096.\n")
+	"Valid values are 512 or 4096. The recommended value is 4096.\n")
 
 cfg(allocation_vdo_block_map_cache_size_mb_CFG, "vdo_block_map_cache_size_mb", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BLOCK_MAP_CACHE_SIZE_MB, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the amount of memory in MiB allocated for caching block map\n"
@@ -726,7 +729,8 @@ cfg(allocation_vdo_block_map_era_length_CFG, "vdo_block_map_period", allocation_
 	"The speed with which the block map cache writes out modified block map pages.\n"
 	"A smaller era length is likely to reduce the amount time spent rebuilding,\n"
 	"at the cost of increased block map writes during normal operation.\n"
-	"The maximum and recommended value is 16380; the minimum value is 1.\n")
+	"The maximum and recommended value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MAXIMUM)
+	"; the minimum value is " DM_TO_STRING(DM_VDO_BLOCK_MAP_ERA_LENGTH_MINIMUM) ".\n")
 
 cfg(allocation_vdo_check_point_frequency_CFG, "vdo_check_point_frequency", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CHECK_POINT_FREQUENCY, VDO_1ST_VSN, NULL, 0, NULL,
 	"The default check point frequency for VDO volume.\n")
@@ -748,27 +752,34 @@ cfg(allocation_vdo_slab_size_mb_CFG, "vdo_slab_size_mb", allocation_CFG_SECTION,
 cfg(allocation_vdo_ack_threads_CFG, "vdo_ack_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_ACK_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for acknowledging\n"
 	"completion of requested VDO I/O operations.\n"
-	"The value must be at in range [0..100].\n")
+	"The value must be at in range [" DM_TO_STRING(DM_VDO_ACK_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_ACK_THREADS_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_bio_threads_CFG, "vdo_bio_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BIO_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for submitting I/O\n"
 	"operations to the storage device of VDO volume.\n"
-	"The value must be in range [1..100]\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_BIO_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_BIO_THREADS_MAXIMUM) "].\n"
 	"Each additional thread after the first will use an additional 18MiB of RAM,\n"
 	"plus 1.12 MiB of RAM per megabyte of configured read cache size.\n")
 
 cfg(allocation_vdo_bio_rotation_CFG, "vdo_bio_rotation", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_BIO_ROTATION, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of I/O operations to enqueue for each bio-submission\n"
-	"thread before directing work to the next. The value must be in range [1..1024].\n")
+	"thread before directing work to the next. The value must be in range ["
+	DM_TO_STRING(DM_VDO_BIO_ROTATION_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_BIO_ROTATION_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_cpu_threads_CFG, "vdo_cpu_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_CPU_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads to use for CPU-intensive work such as\n"
-	"hashing or compression for VDO volume. The value must be in range [1..100]\n")
+	"hashing or compression for VDO volume. The value must be in range ["
+	DM_TO_STRING(DM_VDO_CPU_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_CPU_THREADS_MAXIMUM) "].\n")
 
 cfg(allocation_vdo_hash_zone_threads_CFG, "vdo_hash_zone_threads", allocation_CFG_SECTION, CFG_PROFILABLE | CFG_PROFILABLE_METADATA | CFG_DEFAULT_COMMENTED, CFG_TYPE_INT, DEFAULT_VDO_HASH_ZONE_THREADS, VDO_1ST_VSN, NULL, 0, NULL,
 	"Specifies the number of threads across which to subdivide parts of the VDO\n"
 	"processing based on the hash value computed from the block data.\n"
-	"The value must be at in range [0..100].\n"
+	"The value must be at in range [" DM_TO_STRING(DM_VDO_HASH_ZONE_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_HASH_ZONE_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -777,7 +788,8 @@ cfg(allocation_vdo_logical_threads_CFG, "vdo_logical_threads", allocation_CFG_SE
 	"processing based on the hash value computed from the block data.\n"
 	"A logical thread count of 9 or more will require explicitly specifying\n"
 	"a sufficiently large block map cache size, as well.\n"
-	"The value must be in range [0..100].\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_LOGICAL_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_LOGICAL_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -785,7 +797,8 @@ cfg(allocation_vdo_physical_threads_CFG, "vdo_physical_threads", allocation_CFG_
 	"Specifies the number of threads across which to subdivide parts of the VDO\n"
 	"processing based on physical block addresses.\n"
 	"Each additional thread after the first will use an additional 10MiB of RAM.\n"
-	"The value must be in range [0..16].\n"
+	"The value must be in range [" DM_TO_STRING(DM_VDO_PHYSICAL_THREADS_MINIMUM) ".."
+	DM_TO_STRING(DM_VDO_PHYSICAL_THREADS_MAXIMUM) "].\n"
 	"vdo_hash_zone_threads, vdo_logical_threads and vdo_physical_threads must be\n"
 	"either all zero or all non-zero.\n")
 
@@ -1796,7 +1809,14 @@ cfg(report_output_format_CFG, "output_format", report_CFG_SECTION, CFG_PROFILABL
 	"    one report per command, each report is prefixed with report's\n"
 	"    name for identification.\n"
 	"  json\n"
-	"    JSON format.\n")
+	"    JSON format.\n"
+	"  json_std\n"
+	"    JSON format that is more compliant with JSON standard.\n"
+        "    Compared to original \"json\" format:\n"
+        "      - it does not use double quotes around numeric values,\n"
+        "      - it uses 'null' for undefined numeric values,\n"
+        "      - it prints string list as proper JSON array of strings instead of a single string."
+	"\n")
 
 cfg(report_compact_output_CFG, "compact_output", report_CFG_SECTION, CFG_PROFILABLE | CFG_DEFAULT_COMMENTED, CFG_TYPE_BOOL, DEFAULT_REP_COMPACT_OUTPUT, vsn(2, 2, 115), NULL, 0, NULL,
 	"Do not print empty values for all report fields.\n"

lib/config/defaults.h

@@ -332,4 +332,6 @@
 #define VGS_ONLINE_DIR DEFAULT_RUN_DIR "/vgs_online"
 #define PVS_LOOKUP_DIR DEFAULT_RUN_DIR "/pvs_lookup"
 
+#define DEFAULT_DEVICE_ID_SYSFS_DIR "/sys/"  /* trailing / to match dm_sysfs_dir() */
+
 #endif				/* _LVM_DEFAULTS_H */

lib/device/dev-cache.c

@@ -52,6 +52,7 @@ static struct {
 	struct dm_regex *preferred_names_matcher;
 	const char *dev_dir;
 
+	int preferred_names_disabled;
 	int has_scanned;
 	long st_dev;
 	struct dm_list dirs;
@@ -80,6 +81,7 @@ static void _dev_init(struct device *dev)
 
 	dm_list_init(&dev->aliases);
 	dm_list_init(&dev->ids);
+	dm_list_init(&dev->wwids);
 }
 
 void dev_destroy_file(struct device *dev)
@@ -165,11 +167,19 @@ void dev_set_preferred_name(struct dm_str_list *sl, struct device *dev)
 	if (_cache.preferred_names_matcher)
 		return;
 
+	if (_cache.preferred_names_disabled)
+		return;
+
 	log_debug_devs("%s: New preferred name", sl->str);
 	dm_list_del(&sl->list);
 	dm_list_add_h(&dev->aliases, &sl->list);
 }
 
+void dev_cache_disable_preferred_names(void)
+{
+	_cache.preferred_names_disabled = 1;
+}
+
 /*
  * Check whether path0 or path1 contains the subpath. The path that
  * *does not* contain the subpath wins (return 0 or 1). If both paths
@@ -383,6 +393,22 @@ out:
 	return 1;
 }
 
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen)
+{
+	int ret;
+	int fd;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return 0;
+	ret = read(fd, buf, buf_size);
+	close(fd);
+	if (ret <= 0)
+		return 0;
+	*retlen = ret;
+	return 1;
+}
+
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value)
 {
 	FILE *fp;
@@ -1336,6 +1362,7 @@ int dev_cache_exit(void)
 		dm_hash_iterate(n, _cache.names) {
 			dev = (struct device *) dm_hash_get_data(_cache.names, n);
 			free_dids(&dev->ids);
+			free_wwids(&dev->wwids);
 		}
 	}
 
@@ -1410,7 +1437,7 @@ static void _remove_alias(struct device *dev, const char *name)
  * deactivated LV.  Those old paths are all invalid and are dropped here.
  */
 
-static void _verify_aliases(struct device *dev)
+void dev_cache_verify_aliases(struct device *dev)
 {
 	struct dm_str_list *strl, *strl2;
 	struct stat st;
@@ -1459,7 +1486,7 @@ static struct device *_dev_cache_get(struct cmd_context *cmd, const char *name,
 			_remove_alias(dev, name);
 
 			/* Remove any other names in dev->aliases that are incorrect. */
-			_verify_aliases(dev);
+			dev_cache_verify_aliases(dev);
 		}
 		return NULL;
 	}
@@ -1606,18 +1633,6 @@ static struct device *_dev_cache_get(struct cmd_context *cmd, const char *name,
 		return dev;
 
 	ret = f->passes_filter(cmd, f, dev, NULL);
-
-	/*
-	 * This might happen if this function is called before
-	 * filters can do i/o.  I don't think this will happen
-	 * any longer and this EAGAIN case can be removed.
-	 */
-	if (ret == -EAGAIN) {
-		log_debug_devs("dev_cache_get filter deferred %s", dev_name(dev));
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		ret = 1;
-	}
-
 	if (!ret) {
 		log_debug_devs("dev_cache_get filter excludes %s", dev_name(dev));
 		return NULL;
@@ -1688,16 +1703,9 @@ struct device *dev_iter_get(struct cmd_context *cmd, struct dev_iter *iter)
 
 		f = iter->filter;
 
-		if (f && !(d->flags & DEV_REGULAR)) {
+		if (f && !(d->flags & DEV_REGULAR))
 			ret = f->passes_filter(cmd, f, d, NULL);
 
-			if (ret == -EAGAIN) {
-				log_debug_devs("get device by iter defer filter %s", dev_name(d));
-				d->flags |= DEV_FILTER_AFTER_SCAN;
-				ret = 1;
-			}
-		}
-
 		if (!f || (d->flags & DEV_REGULAR) || ret)
 			return d;
 	}
@@ -1882,6 +1890,15 @@ int setup_devices(struct cmd_context *cmd)
 
 	file_exists = devices_file_exists(cmd);
 
+	/*
+	 * Fail if user specifies a file name that doesn't exist and
+	 * the command is not creating a new devices file.
+	 */
+	if (!file_exists && !cmd->create_edit_devices_file && cmd->devicesfile && strlen(cmd->devicesfile)) {
+		log_error("Devices file not found: %s", cmd->devices_file_path);
+		return 0;
+	}
+
 	/*
 	 * Removing the devices file is another way of disabling the use of
 	 * a devices file, unless the command creates the devices file.
@@ -1929,10 +1946,9 @@ int setup_devices(struct cmd_context *cmd)
 
 	if (!file_exists) {
 		/*
-		 * pvcreate/vgcreate/vgimportdevices/lvmdevices-add create
-		 * a new devices file here if it doesn't exist.
-		 * They have the create_edit_devices_file flag set.
-		 * First they create/lock-ex the devices file lockfile.
+		 * pvcreate/vgcreate create a new devices file here if it
+		 * doesn't exist.  They have create_edit_devices_file=1.
+		 * First create/lock-ex the devices file lockfile.
 		 * Other commands will not use a devices file if none exists.
 		 */
 		lock_mode = LOCK_EX;

lib/device/dev-cache.h

@@ -55,11 +55,14 @@ int dev_cache_add_dir(const char *path);
 struct device *dev_cache_get(struct cmd_context *cmd, const char *name, struct dev_filter *f);
 struct device *dev_cache_get_existing(struct cmd_context *cmd, const char *name, struct dev_filter *f);
 struct device *dev_cache_get_by_devt(struct cmd_context *cmd, dev_t devt);
+void dev_cache_verify_aliases(struct device *dev);
 
 struct device *dev_hash_get(const char *name);
 
 void dev_set_preferred_name(struct dm_str_list *sl, struct device *dev);
 
+void dev_cache_disable_preferred_names(void);
+
 /*
  * Object for iterating through the cache.
  */
@@ -73,6 +76,7 @@ void dev_cache_failed_path(struct device *dev, const char *path);
 bool dev_cache_has_md_with_end_superblock(struct dev_types *dt);
 
 int get_sysfs_value(const char *path, char *buf, size_t buf_size, int error_if_no_value);
+int get_sysfs_binary(const char *path, char *buf, size_t buf_size, int *retlen);
 int get_dm_uuid_from_sysfs(char *buf, size_t buf_size, int major, int minor);
 
 int setup_devices_file(struct cmd_context *cmd);

lib/device/dev-luks.c

@@ -23,9 +23,6 @@ int dev_is_luks(struct cmd_context *cmd, struct device *dev, uint64_t *offset_fo
 	char buf[LUKS_SIGNATURE_SIZE];
 	int ret = -1;
 
-	if (!scan_bcache)
-		return -EAGAIN;
-
 	if (offset_found)
 		*offset_found = 0;
 

lib/device/dev-md.c

@@ -178,12 +178,6 @@ static int _dev_is_md_component_native(struct device *dev, uint64_t *offset_foun
 	uint64_t size, sb_offset = 0;
 	int ret;
 
-	/* i/o layer has not been set up */
-	if (!scan_bcache) {
-		log_error(INTERNAL_ERROR "dev_is_md_component_native requires io layer.");
-		return -1;
-	}
-
 	if (!dev_get_size(dev, &size)) {
 		stack;
 		return -1;

lib/device/dev-mpath.c

@@ -17,12 +17,14 @@
 #include "lib/activate/activate.h"
 #include "lib/commands/toolcontext.h"
 #include "lib/device/device_id.h"
+#include "lib/datastruct/str_list.h"
 #ifdef UDEV_SYNC_SUPPORT
 #include <libudev.h>
 #include "lib/device/dev-ext-udev-constants.h"
 #endif
 
 #include <dirent.h>
+#include <ctype.h>
 
 #define MPATH_PREFIX "mpath-"
 
@@ -35,21 +37,175 @@
  * If dm-3 is not an mpath device, then the constant "1" is stored in
  * the hash table with the key of the dm minor number.
  */
-static struct dm_pool *_hash_mem;
+static struct dm_pool *_wwid_mem;
 static struct dm_hash_table *_minor_hash_tab;
 static struct dm_hash_table *_wwid_hash_tab;
+static struct dm_list _ignored;
+static struct dm_list _ignored_exceptions;
 
 #define MAX_WWID_LINE 512
 
-/*
- * do we need to check the multipath.conf blacklist?
- */
+static void _read_blacklist_file(const char *path)
+{
+	FILE *fp;
+	char line[MAX_WWID_LINE];
+	char wwid[MAX_WWID_LINE];
+	char *word, *p;
+	int section_black = 0;
+	int section_exceptions = 0;
+	int found_quote;
+	int found_type;
+	int i, j;
 
-static void _read_wwid_file(const char *config_wwids_file)
+	if (!(fp = fopen(path, "r")))
+		return;
+
+	while (fgets(line, sizeof(line), fp)) {
+		word = NULL;
+
+		/* skip initial white space on the line */
+		for (i = 0; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (isspace(line[i]))
+				continue;
+			word = &line[i];
+			break;
+		}
+
+		if (!word || word[0] == '#')
+			continue;
+
+		/* identify the start of the section we want to read */
+		if (strchr(word, '{')) {
+			if (!strncmp(word, "blacklist_exceptions", 20))
+				section_exceptions = 1;
+			else if (!strncmp(word, "blacklist", 9))
+				section_black = 1;
+			continue;
+		}
+		/* identify the end of the section we've been reading */
+		if (strchr(word, '}')) {
+			section_exceptions = 0;
+			section_black = 0;
+			continue;
+		}
+		/* skip lines that are not in a section we want */
+		if (!section_black && !section_exceptions)
+			continue;
+
+		/*
+		 * read a wwid from the blacklist{_exceptions} section.
+		 * does not recognize other non-wwid entries in the
+		 * section, and skips those (should the entire mp
+		 * config filtering be disabled if non-wwids are seen?
+		 */
+		if (!(p = strstr(word, "wwid")))
+			continue;
+
+		i += 4; /* skip "wwid" */
+
+		/*
+		 * copy wwid value from the line.
+		 * the wwids copied here need to match the
+		 * wwids read from /etc/multipath/wwids,
+		 * which are matched to wwids from sysfs.
+		 */
+
+		memset(wwid, 0, sizeof(wwid));
+		found_quote = 0;
+		found_type = 0;
+		j = 0;
+
+		for (; i < MAX_WWID_LINE; i++) {
+			if ((line[i] == '\n') || (line[i] == '\0'))
+				break;
+			if (!j && isspace(line[i]))
+				continue;
+			if (isspace(line[i]))
+				break;
+			/* quotes around wwid are optional */
+			if ((line[i] == '"') && !found_quote) {
+				found_quote = 1;
+				continue;
+			}
+			/* second quote is end of wwid */
+			if ((line[i] == '"') && found_quote)
+				break;
+			/* exclude initial 3/2/1 for naa/eui/t10 */
+			if (!j && !found_type &&
+			    ((line[i] == '3') || (line[i] == '2') || (line[i] == '1'))) {
+				found_type = 1;
+				continue;
+			}
+
+			wwid[j] = line[i];
+			j++;
+		}
+
+		if (j < 8)
+			continue;
+
+		log_debug("multipath wwid %s in %s %s",
+			  wwid, section_exceptions ? "blacklist_exceptions" : "blacklist", path);
+
+		if (section_exceptions) {
+			if (!str_list_add(_wwid_mem, &_ignored_exceptions, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		} else {
+			if (!str_list_add(_wwid_mem, &_ignored, dm_pool_strdup(_wwid_mem, wwid)))
+				stack;
+		}
+	}
+
+	if (fclose(fp))
+		stack;
+}
+
+static void _read_wwid_exclusions(void)
+{
+	char path[PATH_MAX] = { 0 };
+	DIR *dir;
+	struct dirent *de;
+	struct dm_str_list *sl, *sl2;
+	int rem_count = 0;
+
+	_read_blacklist_file("/etc/multipath.conf");
+
+	if ((dir = opendir("/etc/multipath/conf.d"))) {
+		while ((de = readdir(dir))) {
+			if (de->d_name[0] == '.')
+				continue;
+			snprintf(path, PATH_MAX-1, "/etc/multipath/conf.d/%s", de->d_name);
+			_read_blacklist_file(path);
+		}
+		closedir(dir);
+	}
+
+	/* for each wwid in ignored_exceptions, remove it from ignored */
+
+	dm_list_iterate_items_safe(sl, sl2, &_ignored) {
+		if (str_list_match_item(&_ignored_exceptions, sl->str))
+			str_list_del(&_ignored, sl->str);
+	}
+
+	/* for each wwid in ignored, remove it from wwid_hash */
+
+	dm_list_iterate_items(sl, &_ignored) {
+		dm_hash_remove_binary(_wwid_hash_tab, sl->str, strlen(sl->str));
+		rem_count++;
+	}
+
+	if (rem_count)
+		log_debug("multipath config ignored %d wwids", rem_count);
+}
+
+static void _read_wwid_file(const char *config_wwids_file, int *entries)
 {
 	FILE *fp;
 	char line[MAX_WWID_LINE];
 	char *wwid, *p;
+	char typestr[2] = { 0 };
 	int count = 0;
 
 	if (config_wwids_file[0] != '/') {
@@ -71,8 +227,17 @@ static void _read_wwid_file(const char *config_wwids_file)
 		if (line[0] == '/')
 			wwid++;
 
-		/* skip the initial '3' */
-		wwid++;
+
+		/*
+		 * the initial character is the id type,
+		 * 1 is t10, 2 is eui, 3 is naa, 8 is scsi name.
+		 * wwids are stored in the hash table without the type charater.
+		 * It seems that sometimes multipath does not include
+		 * the type charater (seen with t10 scsi_debug devs).
+		 */
+		typestr[0] = *wwid;
+		if (typestr[0] == '1' || typestr[0] == '2' || typestr[0] == '3')
+			wwid++;
 
 		if ((p = strchr(wwid, '/')))
 			*p = '\0';
@@ -85,6 +250,7 @@ static void _read_wwid_file(const char *config_wwids_file)
 		stack;
 
 	log_debug("multipath wwids read %d from %s", count, config_wwids_file);
+	*entries = count;
 }
 
 int dev_mpath_init(const char *config_wwids_file)
@@ -92,6 +258,10 @@ int dev_mpath_init(const char *config_wwids_file)
 	struct dm_pool *mem;
 	struct dm_hash_table *minor_tab;
 	struct dm_hash_table *wwid_tab;
+	int entries = 0;
+
+	dm_list_init(&_ignored);
+	dm_list_init(&_ignored_exceptions);
 
 	if (!(mem = dm_pool_create("mpath", 256))) {
 		log_error("mpath pool creation failed.");
@@ -104,7 +274,7 @@ int dev_mpath_init(const char *config_wwids_file)
 		return 0;
 	}
 
-	_hash_mem = mem;
+	_wwid_mem = mem;
 	_minor_hash_tab = minor_tab;
 
 	/* multipath_wwids_file="" disables the use of the file */
@@ -116,16 +286,24 @@ int dev_mpath_init(const char *config_wwids_file)
 	if (!(wwid_tab = dm_hash_create(110))) {
 		log_error("mpath hash table creation failed.");
 		dm_hash_destroy(_minor_hash_tab);
-		dm_pool_destroy(_hash_mem);
+		dm_pool_destroy(_wwid_mem);
 		_minor_hash_tab = NULL;
-		_hash_mem = NULL;
+		_wwid_mem = NULL;
 		return 0;
 	}
 
 	_wwid_hash_tab = wwid_tab;
 
-	if (config_wwids_file)
-		_read_wwid_file(config_wwids_file);
+	if (config_wwids_file) {
+		_read_wwid_file(config_wwids_file, &entries);
+		_read_wwid_exclusions();
+	}
+
+	if (!entries) {
+		/* reading dev wwids is skipped with null wwid_hash_tab */
+		dm_hash_destroy(_wwid_hash_tab);
+		_wwid_hash_tab = NULL;
+	}
 
 	return 1;
 }
@@ -136,12 +314,12 @@ void dev_mpath_exit(void)
 		dm_hash_destroy(_minor_hash_tab);
 	if (_wwid_hash_tab)
 		dm_hash_destroy(_wwid_hash_tab);
-	if (_hash_mem)
-		dm_pool_destroy(_hash_mem);
+	if (_wwid_mem)
+		dm_pool_destroy(_wwid_mem);
 
 	_minor_hash_tab = NULL;
 	_wwid_hash_tab = NULL;
-	_hash_mem = NULL;
+	_wwid_mem = NULL;
 }
 
 
@@ -272,10 +450,12 @@ static int _dev_is_mpath_component_udev(struct device *dev)
 }
 #endif
 
-static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev)
+/* mpath_devno is major:minor of the dm multipath device currently using the component dev. */
+
+static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device *dev,
+					 int primary_result, dev_t primary_dev, dev_t *mpath_devno)
 {
 	struct dev_types *dt = cmd->dev_types;
-	const char *part_name;
 	const char *name;               /* e.g. "sda" for "/dev/sda" */
 	char link_path[PATH_MAX];       /* some obscure, unpredictable sysfs path */
 	char holders_path[PATH_MAX];    /* e.g. "/sys/block/sda/holders/" */
@@ -289,25 +469,15 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	int dm_dev_major;
 	int dm_dev_minor;
 	struct stat info;
-	dev_t primary_dev;
 	int is_mpath_component = 0;
 
-	/* multipathing is only known to exist for SCSI or NVME devices */
-	if (!major_is_scsi_device(dt, dev_major) && !dev_is_nvme(dt, dev))
-		return 0;
-
-	switch (dev_get_primary_dev(dt, dev, &primary_dev)) {
+	switch (primary_result) {
 
 	case 2: /* The dev is partition. */
-		part_name = dev_name(dev); /* name of original dev for log_debug msg */
 
 		/* gets "foo" for "/dev/foo" where "/dev/foo" comes from major:minor */
 		if (!(name = _get_sysfs_name_by_devt(sysfs_dir, primary_dev, link_path, sizeof(link_path))))
 			return_0;
-
-		log_debug_devs("%s: Device is a partition, using primary "
-			       "device %s for mpath component detection",
-			       part_name, name);
 		break;
 
 	case 1: /* The dev is already a primary dev. Just continue with the dev. */
@@ -426,50 +596,104 @@ static int _dev_is_mpath_component_sysfs(struct cmd_context *cmd, struct device
 	if (closedir(dr))
 		stack;
 
+	if (is_mpath_component)
+		*mpath_devno = MKDEV(dm_dev_major, dm_dev_minor);
 	return is_mpath_component;
 }
 
-static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev)
+static int _dev_in_wwid_file(struct cmd_context *cmd, struct device *dev,
+			     int primary_result, dev_t primary_dev)
 {
-	char sysbuf[PATH_MAX] = { 0 };
-	char *wwid;
-	long look;
+	char idbuf[DEV_WWID_SIZE] = { 0 };
+	struct dev_wwid *dw;
+	char *wwid, *full_wwid;
 
 	if (!_wwid_hash_tab)
 		return 0;
 
-	if (!read_sys_block(cmd, dev, "device/wwid", sysbuf, sizeof(sysbuf)))
-		return 0;
+	/*
+	 * Check the primary device, not the partition.
+	 */
+	if (primary_result == 2) {
+		if (!(dev = dev_cache_get_by_devt(cmd, primary_dev))) {
+			log_debug("dev_is_mpath_component %s no primary dev", dev_name(dev));
+			return 0;
+		}
+	}
 
-	if (!sysbuf[0])
+	/*
+	 * sysfs wwid uses format: naa.<value>, eui.<value>, t10.<value>.
+	 * multipath wwids file uses format: 3<value>, 2<value>, 1<value>.
+	 *
+	 * We omit the type prefix before looking up.  The multipath/wwids
+	 * values in the wwid_hash_tab have the initial character removed.
+	 *
+	 * There's no type prefix for "scsi name string" type 8 ids.
+	 *
+	 * First try looking up any wwids that have already been read.
+	 */
+lookup:
+	dm_list_iterate_items(dw, &dev->wwids) {
+		if (dw->type == 1 || dw->type == 2 || dw->type == 3)
+			wwid = &dw->id[4];
+		else
+			wwid = dw->id;
+
+		if (dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid))) {
+			full_wwid = dw->id;
+			goto found;
+		}
+	}
+
+	/*
+	 * The id from sysfs wwid may not be the id used by multipath,
+	 * or a device may not have a vpd_pg83 file (e.g. nvme).
+	 */
+
+	if (!(dev->flags & DEV_ADDED_VPD_WWIDS) && dev_read_vpd_wwids(cmd, dev))
+		goto lookup;
+
+	if (!(dev->flags & DEV_ADDED_SYS_WWID) && dev_read_sys_wwid(cmd, dev, idbuf, sizeof(idbuf), &dw)) {
+		if (dw->type == 1 || dw->type == 2 || dw->type == 3)
+			wwid = &dw->id[4];
+		else
+			wwid = dw->id;
+
+		if (dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid))) {
+			full_wwid = dw->id;
+			goto found;
+		}
+	}
+
+	return 0;
+
+ found:
+	log_debug_devs("dev_is_mpath_component %s %s in wwids file", dev_name(dev), full_wwid);
+	return 1;
+}
+
+int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev, dev_t *holder_devno)
+{
+	struct dev_types *dt = cmd->dev_types;
+	int primary_result;
+	dev_t primary_dev;
+
+	/*
+	 * multipath only uses SCSI or NVME devices
+	 */
+	if (!major_is_scsi_device(dt, MAJOR(dev->dev)) && !dev_is_nvme(dt, dev))
 		return 0;
 
 	/*
-	 * sysfs prints wwid as <typestr>.<value>
-	 * multipath wwid uses '3'<value>
-	 * does "<typestr>." always correspond to "3"?
+	 * primary_result 2: dev is a partition, primary_dev is the whole device
+	 * primary_result 1: dev is a whole device
 	 */
-	if (!(wwid = strchr(sysbuf, '.')))
-		return 0;
+	primary_result = dev_get_primary_dev(dt, dev, &primary_dev);
 
-	/* skip the type and dot, just as '3' was skipped from wwids entry */
-	wwid++;
-	
-	look = (long) dm_hash_lookup_binary(_wwid_hash_tab, wwid, strlen(wwid));
-
-	if (look) {
-		log_debug_devs("dev_is_mpath_component %s multipath wwid %s", dev_name(dev), wwid);
-		return 1;
-	}
-	return 0;
-}
-
-int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
-{
-	if (_dev_is_mpath_component_sysfs(cmd, dev) == 1)
+	if (_dev_is_mpath_component_sysfs(cmd, dev, primary_result, primary_dev, holder_devno) == 1)
 		goto found;
 
-	if (_dev_in_wwid_file(cmd, dev))
+	if (_dev_in_wwid_file(cmd, dev, primary_result, primary_dev))
 		goto found;
 
 	if (external_device_info_source() == DEV_EXT_UDEV) {
@@ -477,6 +701,12 @@ int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev)
 			goto found;
 	}
 
+	/*
+	 * TODO: save the result of this function in dev->flags and use those
+	 * flags on repeated calls to avoid repeating the work multiple times
+	 * for the same device when there are partitions on the device.
+	 */
+
 	return 0;
 found:
 	return 1;
@@ -495,7 +725,7 @@ const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev
 
 	/* /sys/dev/block/253:7/slaves/sda/device/wwid */
 
-	if (dm_snprintf(slaves_path, sizeof(slaves_path), "%s/dev/block/%d:%d/slaves",
+	if (dm_snprintf(slaves_path, sizeof(slaves_path), "%sdev/block/%d:%d/slaves",
 			dm_sysfs_dir(), (int)MAJOR(dev->dev), (int)MINOR(dev->dev)) < 0) {
 		log_warn("Sysfs path to check mpath components is too long.");
 		return NULL;
@@ -525,7 +755,7 @@ const char *dev_mpath_component_wwid(struct cmd_context *cmd, struct device *dev
 
 		/* read /sys/block/sda/device/wwid */
 
-		if (dm_snprintf(wwid_path, sizeof(wwid_path), "%s/block/%s/device/wwid",
+		if (dm_snprintf(wwid_path, sizeof(wwid_path), "%sblock/%s/device/wwid",
        				dm_sysfs_dir(), slave_name) < 0) {
 			log_warn("Failed to create sysfs wwid path for %s", slave_name);
 			continue;

lib/device/dev-swap.c

@@ -42,9 +42,6 @@ int dev_is_swap(struct cmd_context *cmd, struct device *dev, uint64_t *offset_fo
 	unsigned page;
 	int ret = 0;
 
-	if (!scan_bcache)
-		return -EAGAIN;
-
 	if (!dev_get_size(dev, &size)) {
 		stack;
 		return -1;

lib/device/dev-type.c

@@ -25,7 +25,7 @@
 #include "device_mapper/misc/dm-ioctl.h"
 
 #ifdef BLKID_WIPING_SUPPORT
-#include <blkid.h>
+#include <blkid/blkid.h>
 #endif
 
 #ifdef UDEV_SYNC_SUPPORT
@@ -548,7 +548,7 @@ static int _has_sys_partition(struct device *dev)
 	int minor = (int) MINOR(dev->dev);
 
 	/* check if dev is a partition */
-	if (dm_snprintf(path, sizeof(path), "%s/dev/block/%d:%d/partition",
+	if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/partition",
 			dm_sysfs_dir(), major, minor) < 0) {
 		log_warn("WARNING: %s: partition path is too long.", dev_name(dev));
 		return 0;
@@ -674,11 +674,6 @@ static int _dev_is_partitioned_native(struct dev_types *dt, struct device *dev)
 {
 	int r;
 
-	if (!scan_bcache) {
-		log_error(INTERNAL_ERROR "dev_is_partitioned_native requires i/o.");
-		return -1;
-	}
-
 	/* Unpartitioned DASD devices are not supported. */
 	if ((MAJOR(dev->dev) == dt->dasd_major) && dasd_is_cdl_formatted(dev))
 		return 1;
@@ -780,7 +775,7 @@ int dev_get_primary_dev(struct dev_types *dt, struct device *dev, dev_t *result)
 	 * - basename ../../block/md0/md0  = md0
 	 * Parent's 'dev' sysfs attribute  = /sys/block/md0/dev
 	 */
-	if (dm_snprintf(path, sizeof(path), "%s/dev/block/%d:%d",
+	if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d",
 			dm_sysfs_dir(), major, minor) < 0) {
 		log_warn("WARNING: %s: major:minor sysfs path is too long.", dev_name(dev));
 		return 0;
@@ -792,7 +787,7 @@ int dev_get_primary_dev(struct dev_types *dt, struct device *dev, dev_t *result)
 
 	temp_path[size] = '\0';
 
-	if (dm_snprintf(path, sizeof(path), "%s/block/%s/dev",
+	if (dm_snprintf(path, sizeof(path), "%sblock/%s/dev",
 			dm_sysfs_dir(), basename(dirname(temp_path))) < 0) {
 		log_warn("WARNING: sysfs path for %s is too long.",
 			 basename(dirname(temp_path)));
@@ -828,26 +823,143 @@ out:
 }
 
 #ifdef BLKID_WIPING_SUPPORT
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size)
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs)
 {
-	char *block_size_str = NULL;
+	blkid_probe probe = NULL;
+	const char *type_str = NULL, *size_str = NULL;
+	size_t len;
+	int ret = 1;
+	int rc;
 
-	if ((block_size_str = blkid_get_tag_value(NULL, "BLOCK_SIZE", pathname))) {
-		*fs_block_size = (uint32_t)atoi(block_size_str);
-		free(block_size_str);
-		log_debug("Found blkid BLOCK_SIZE %u for fs on %s", *fs_block_size, pathname);
-		return 1;
-	} else {
-		log_debug("No blkid BLOCK_SIZE for fs on %s", pathname);
-		*fs_block_size = 0;
+	if (!(probe = blkid_new_probe_from_filename(pathname))) {
+		log_error("Failed libblkid probe setup for %s", pathname);
 		return 0;
 	}
+
+	blkid_probe_enable_superblocks(probe, 1);
+	blkid_probe_set_superblocks_flags(probe,
+			BLKID_SUBLKS_LABEL | BLKID_SUBLKS_LABELRAW |
+			BLKID_SUBLKS_UUID | BLKID_SUBLKS_UUIDRAW |
+			BLKID_SUBLKS_TYPE | BLKID_SUBLKS_SECTYPE |
+			BLKID_SUBLKS_USAGE | BLKID_SUBLKS_VERSION |
+#ifdef BLKID_SUBLKS_FSINFO
+			BLKID_SUBLKS_FSINFO |
+#endif
+			BLKID_SUBLKS_MAGIC);
+	rc = blkid_do_safeprobe(probe);
+	if (rc < 0) {
+		log_debug("Failed libblkid probe for %s", pathname);
+		ret = 0;
+		goto out;
+	} else if (rc == 1) {
+		/* no file system on the device */
+		log_debug("No file system found on %s.", pathname);
+		if (nofs)
+			*nofs = 1;
+		goto out;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "TYPE", &type_str, &len) && len && type_str) {
+		if (fstype)
+			strncpy(fstype, type_str, FSTYPE_MAX);
+	} else {
+		/* any difference from blkid_do_safeprobe rc=1? */
+		log_debug("No file system type on %s.", pathname);
+		if (nofs)
+			*nofs = 1;
+		goto out;
+	}
+
+	if (fs_block_size_bytes) {
+		if (!blkid_probe_lookup_value(probe, "BLOCK_SIZE", &size_str, &len) && len && size_str)
+			*fs_block_size_bytes = atoi(size_str);
+		else
+			*fs_block_size_bytes = 0;
+	}
+
+	log_debug("Found blkid fstype %s fsblocksize %s on %s",
+		  type_str ?: "none", size_str ?: "unused", pathname);
+out:
+	blkid_free_probe(probe);
+	return ret;
 }
+
+int fs_get_blkid(const char *pathname, struct fs_info *fsi)
+{
+	blkid_probe probe = NULL;
+	const char *str;
+	size_t len;
+	uint64_t fslastblock = 0;
+	unsigned int fsblocksize = 0;
+	int rc;
+
+	if (!(probe = blkid_new_probe_from_filename(pathname))) {
+		log_error("Failed libblkid probe setup for %s", pathname);
+		return 0;
+	}
+
+	blkid_probe_enable_superblocks(probe, 1);
+	blkid_probe_set_superblocks_flags(probe,
+			BLKID_SUBLKS_LABEL | BLKID_SUBLKS_LABELRAW |
+			BLKID_SUBLKS_UUID | BLKID_SUBLKS_UUIDRAW |
+			BLKID_SUBLKS_TYPE | BLKID_SUBLKS_SECTYPE |
+			BLKID_SUBLKS_USAGE | BLKID_SUBLKS_VERSION |
+#ifdef BLKID_SUBLKS_FSINFO
+			BLKID_SUBLKS_FSINFO |
+#endif
+			BLKID_SUBLKS_MAGIC);
+	rc = blkid_do_safeprobe(probe);
+	if (rc < 0) {
+		log_error("Failed libblkid probe for %s", pathname);
+		blkid_free_probe(probe);
+		return 0;
+	} else if (rc == 1) {
+		/* no file system on the device */
+		log_print("No file system found on %s.", pathname);
+		fsi->nofs = 1;
+		blkid_free_probe(probe);
+		return 1;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "TYPE", &str, &len) && len)
+		strncpy(fsi->fstype, str, sizeof(fsi->fstype)-1);
+	else {
+		/* any difference from blkid_do_safeprobe rc=1? */
+		log_print("No file system type on %s.", pathname);
+		fsi->nofs = 1;
+		blkid_free_probe(probe);
+		return 1;
+	}
+
+	if (!blkid_probe_lookup_value(probe, "BLOCK_SIZE", &str, &len) && len)
+		fsi->fs_block_size_bytes = atoi(str);
+
+	if (!blkid_probe_lookup_value(probe, "FSLASTBLOCK", &str, &len) && len)
+		fslastblock = strtoull(str, NULL, 0);
+
+	if (!blkid_probe_lookup_value(probe, "FSBLOCKSIZE", &str, &len) && len)
+		fsblocksize = (unsigned int)atoi(str);
+
+	blkid_free_probe(probe);
+
+	if (fslastblock && fsblocksize)
+		fsi->fs_last_byte = fslastblock * fsblocksize;
+
+	log_debug("libblkid TYPE %s BLOCK_SIZE %d FSLASTBLOCK %llu FSBLOCKSIZE %u fs_last_byte %llu",
+		  fsi->fstype, fsi->fs_block_size_bytes, (unsigned long long)fslastblock, fsblocksize,
+		  (unsigned long long)fsi->fs_last_byte);
+	return 1;
+}
+
 #else
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size)
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs)
 {
 	log_debug("Disabled blkid BLOCK_SIZE for fs.");
-	*fs_block_size = 0;
+	return 0;
+}
+int fs_get_blkid(const char *pathname, struct fs_info *fsi)
+{
+	log_debug("Disabled blkid for fs info.");
 	return 0;
 }
 #endif
@@ -1100,7 +1212,7 @@ int wipe_known_signatures(struct cmd_context *cmd, struct device *dev,
 static int _snprintf_attr(char *buf, size_t buf_size, const char *sysfs_dir,
 			 const char *attribute, dev_t dev)
 {
-	if (dm_snprintf(buf, buf_size, "%s/dev/block/%d:%d/%s", sysfs_dir,
+	if (dm_snprintf(buf, buf_size, "%sdev/block/%d:%d/%s", sysfs_dir,
 			(int)MAJOR(dev), (int)MINOR(dev),
 			attribute) < 0) {
 		log_warn("WARNING: sysfs path for %s attribute is too long.", attribute);

lib/device/dev-type.h

@@ -18,6 +18,7 @@
 #include "lib/device/device.h"
 #include "lib/display/display.h"
 #include "lib/label/label.h"
+#include "lib/device/filesystem.h"
 
 #define NUMBER_OF_MAJORS 4096
 
@@ -58,7 +59,7 @@ int major_is_scsi_device(struct dev_types *dt, int major);
 
 /* Signature/superblock recognition with position returned where found. */
 int dev_is_md_component(struct cmd_context *cmd, struct device *dev, uint64_t *sb, int full);
-int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev);
+int dev_is_mpath_component(struct cmd_context *cmd, struct device *dev, dev_t *mpath_devno);
 int dev_is_swap(struct cmd_context *cmd, struct device *dev, uint64_t *signature, int full);
 int dev_is_luks(struct cmd_context *cmd, struct device *dev, uint64_t *signature, int full);
 int dasd_is_cdl_formatted(struct device *dev);
@@ -101,7 +102,9 @@ int dev_is_nvme(struct dev_types *dt, struct device *dev);
 
 int dev_is_lv(struct device *dev);
 
-int get_fs_block_size(const char *pathname, uint32_t *fs_block_size);
+#define FSTYPE_MAX 16
+int fs_block_size_and_type(const char *pathname, uint32_t *fs_block_size_bytes, char *fstype, int *nofs);
+int fs_get_blkid(const char *pathname, struct fs_info *fsi);
 
 int dev_is_used_by_active_lv(struct cmd_context *cmd, struct device *dev, int *used_by_lv_count,
 			     char **used_by_dm_name, char **used_by_vg_uuid, char **used_by_lv_uuid);

lib/device/dev_util.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2013 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/device/device.h"
+
+int device_id_list_remove(struct dm_list *list, struct device *dev)
+{
+	struct device_id_list *dil;
+
+	dm_list_iterate_items(dil, list) {
+		if (dil->dev == dev) {
+			dm_list_del(&dil->list);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+struct device_id_list *device_id_list_find_dev(struct dm_list *list, struct device *dev)
+{
+	struct device_id_list *dil;
+
+	dm_list_iterate_items(dil, list) {
+		if (dil->dev == dev)
+			return dil;
+	}
+	return NULL;
+}
+
+int device_list_remove(struct dm_list *list, struct device *dev)
+{
+	struct device_list *devl;
+
+	dm_list_iterate_items(devl, list) {
+		if (devl->dev == dev) {
+			dm_list_del(&devl->list);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+struct device_list *device_list_find_dev(struct dm_list *list, struct device *dev)
+{
+	struct device_list *devl;
+
+	dm_list_iterate_items(devl, list) {
+		if (devl->dev == dev)
+			return devl;
+	}
+	return NULL;
+}
+

lib/device/device.h

@@ -32,8 +32,8 @@
 #define DEV_NOT_O_NOATIME	0x00000400	/* Don't use O_NOATIME */
 #define DEV_IN_BCACHE		0x00000800      /* dev fd is open and used in bcache */
 #define DEV_BCACHE_EXCL		0x00001000      /* bcache_fd should be open EXCL */
-#define DEV_FILTER_AFTER_SCAN	0x00002000	/* apply filter after bcache has data */
-#define DEV_FILTER_OUT_SCAN	0x00004000	/* filtered out during label scan */
+#define DEV_ADDED_SYS_WWID	0x00002000      /* wwid has been added from sysfs wwid file */
+#define DEV_ADDED_VPD_WWIDS	0x00004000	/* wwids have been added from vpd_pg83 */
 #define DEV_BCACHE_WRITE	0x00008000      /* bcache_fd is open with RDWR */
 #define DEV_SCAN_FOUND_LABEL	0x00010000      /* label scan read dev and found label */
 #define DEV_IS_MD_COMPONENT	0x00020000	/* device is an md component */
@@ -59,14 +59,33 @@ struct dev_ext {
 	void *handle;
 };
 
-#define DEV_ID_TYPE_SYS_WWID   0x0001
-#define DEV_ID_TYPE_SYS_SERIAL 0x0002
-#define DEV_ID_TYPE_MPATH_UUID 0x0003
-#define DEV_ID_TYPE_MD_UUID    0x0004
-#define DEV_ID_TYPE_LOOP_FILE  0x0005
-#define DEV_ID_TYPE_CRYPT_UUID 0x0006
-#define DEV_ID_TYPE_LVMLV_UUID 0x0007
-#define DEV_ID_TYPE_DEVNAME    0x0008
+#define DEV_ID_TYPE_SYS_WWID   1
+#define DEV_ID_TYPE_SYS_SERIAL 2
+#define DEV_ID_TYPE_MPATH_UUID 3
+#define DEV_ID_TYPE_MD_UUID    4
+#define DEV_ID_TYPE_LOOP_FILE  5
+#define DEV_ID_TYPE_CRYPT_UUID 6
+#define DEV_ID_TYPE_LVMLV_UUID 7
+#define DEV_ID_TYPE_DEVNAME    8
+#define DEV_ID_TYPE_WWID_NAA   9
+#define DEV_ID_TYPE_WWID_EUI  10
+#define DEV_ID_TYPE_WWID_T10  11
+
+/* Max length of WWID_NAA, WWID_EUI, WWID_T10 */
+#define DEV_WWID_SIZE 128
+
+/*
+ * A wwid read from:
+ * /sys/dev/block/%d:%d/device/wwid
+ * /sys/dev/block/%d:%d/wwid
+ * /sys/dev/block/%d:%d/device/vpd_pg83
+ */
+
+struct dev_wwid {
+	struct dm_list list;     /* dev->wwids */
+	int type;                /* 1,2,3 for NAA,EUI,T10 */
+	char id[DEV_WWID_SIZE];  /* includes prefix naa.,eui.,t10. */
+};
 
 /*
  * A device ID of a certain type for a device.
@@ -75,7 +94,7 @@ struct dev_ext {
  */
 
 struct dev_id {
-	struct dm_list list;
+	struct dm_list list;    /* dev->ids */
 	struct device *dev;
 	uint16_t idtype;	/* DEV_ID_TYPE_ */
 	char *idname;		/* id string determined by idtype */
@@ -99,12 +118,18 @@ struct dev_use {
 	char *pvid;
 };
 
+struct dev_use_list {
+	struct dm_list list;
+	struct dev_use *du;
+};
+
 /*
  * All devices in LVM will be represented by one of these.
  * pointer comparisons are valid.
  */
 struct device {
 	struct dm_list aliases;	/* struct dm_str_list */
+	struct dm_list wwids; /* struct dev_wwid, used for multipath component detection */
 	struct dm_list ids; /* struct dev_id, different entries for different idtypes */
 	struct dev_id *id; /* points to the the ids entry being used for this dev */
 	dev_t dev;
@@ -154,6 +179,12 @@ struct device_list {
 	struct device *dev;
 };
 
+struct device_id_list {
+	struct dm_list list;
+	struct device *dev;
+	char pvid[ID_LEN + 1];
+};
+
 struct device_area {
 	struct device *dev;
 	uint64_t start;		/* Bytes */
@@ -206,5 +237,14 @@ void dev_destroy_file(struct device *dev);
 
 int dev_mpath_init(const char *config_wwids_file);
 void dev_mpath_exit(void);
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids);
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes);
+int parse_vpd_serial(const unsigned char *in, char *out, int outsize);
+
+/* dev_util */
+int device_id_list_remove(struct dm_list *devices, struct device *dev);
+struct device_id_list *device_id_list_find_dev(struct dm_list *devices, struct device *dev);
+int device_list_remove(struct dm_list *devices, struct device *dev);
+struct device_list *device_list_find_dev(struct dm_list *devices, struct device *dev);
 
 #endif

lib/device/device_id.h

@@ -28,20 +28,23 @@ int device_ids_use_devname(struct cmd_context *cmd);
 int device_ids_read(struct cmd_context *cmd);
 int device_ids_write(struct cmd_context *cmd);
 int device_id_add(struct cmd_context *cmd, struct device *dev, const char *pvid,
-                  const char *idtype_arg, const char *id_arg);
+                  const char *idtype_arg, const char *id_arg, int use_idtype_only);
 void device_id_pvremove(struct cmd_context *cmd, struct device *dev);
 void device_ids_match(struct cmd_context *cmd);
 int device_ids_match_dev(struct cmd_context *cmd, struct device *dev);
 void device_ids_match_device_list(struct cmd_context *cmd);
 void device_ids_validate(struct cmd_context *cmd, struct dm_list *scanned_devs, int *device_ids_invalid, int noupdate);
 int device_ids_version_unchanged(struct cmd_context *cmd);
+void device_ids_check_serial(struct cmd_context *cmd, struct dm_list *scan_devs, int *update_needed, int noupdate);
 void device_ids_find_renamed_devs(struct cmd_context *cmd, struct dm_list *dev_list, int *search_count, int noupdate);
 const char *device_id_system_read(struct cmd_context *cmd, struct device *dev, uint16_t idtype);
 void device_id_update_vg_uuid(struct cmd_context *cmd, struct volume_group *vg, struct id *old_vg_id);
 
+struct dev_use *get_du_for_devno(struct cmd_context *cmd, dev_t devno);
 struct dev_use *get_du_for_dev(struct cmd_context *cmd, struct device *dev);
 struct dev_use *get_du_for_pvid(struct cmd_context *cmd, const char *pvid);
 struct dev_use *get_du_for_devname(struct cmd_context *cmd, const char *devname);
+struct dev_use *get_du_for_device_id(struct cmd_context *cmd, uint16_t idtype, const char *idname);
 
 char *devices_file_version(void);
 int devices_file_exists(struct cmd_context *cmd);
@@ -56,7 +59,17 @@ void devices_file_exit(struct cmd_context *cmd);
 void unlink_searched_devnames(struct cmd_context *cmd);
 
 int read_sys_block(struct cmd_context *cmd, struct device *dev, const char *suffix, char *sysbuf, int sysbufsize);
+int read_sys_block_binary(struct cmd_context *cmd, struct device *dev,
+			  const char *suffix, char *sysbuf, int sysbufsize, int *retlen);
 
 int dev_has_mpath_uuid(struct cmd_context *cmd, struct device *dev, const char **idname_out);
 
+int wwid_type_to_idtype(int wwid_type);
+int idtype_to_wwid_type(int idtype);
+void free_wwids(struct dm_list *ids);
+struct dev_wwid *dev_add_wwid(char *id, int id_type, struct dm_list *ids);
+int dev_read_vpd_wwids(struct cmd_context *cmd, struct device *dev);
+int dev_read_sys_wwid(struct cmd_context *cmd, struct device *dev,
+		      char *buf, int bufsize, struct dev_wwid **dw_out);
+
 #endif

lib/device/filesystem.c

@@ -0,0 +1,425 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/device/device.h"
+#include "lib/device/dev-type.h"
+#include "lib/misc/lvm-exec.h"
+#include "lib/activate/dev_manager.h"
+
+#include <dirent.h>
+#include <mntent.h>
+#include <sys/ioctl.h>
+
+static const char *_lvresize_fs_helper_path;
+
+static const char *_get_lvresize_fs_helper_path(void)
+{
+	if (!_lvresize_fs_helper_path)
+		_lvresize_fs_helper_path = getenv("LVRESIZE_FS_HELPER_PATH");
+
+	if (!_lvresize_fs_helper_path)
+		_lvresize_fs_helper_path = LVRESIZE_FS_HELPER_PATH; /* from configure, usually in /usr/libexec */
+
+	return _lvresize_fs_helper_path;
+}
+
+/*
+ * Set the path of the dm-crypt device, i.e. /dev/dm-N, that is using the LV.
+ */
+static int _get_crypt_path(dev_t lv_devt, char *lv_path, char *crypt_path)
+{
+	char holders_path[PATH_MAX];
+	char *holder_name;
+	DIR *dr;
+	struct stat st;
+	struct dirent *de;
+	int ret = 0;
+
+	if (dm_snprintf(holders_path, sizeof(holders_path), "%sdev/block/%d:%d/holders",
+			dm_sysfs_dir(), (int)MAJOR(lv_devt), (int)MINOR(lv_devt)) < 0)
+		return_0;
+
+	/* If the crypt dev is not active, there will be no LV holder. */
+	if (stat(holders_path, &st)) {
+		log_error("Missing %s for %s", crypt_path, lv_path);
+		return 0;
+	}
+
+	if (!(dr = opendir(holders_path))) {
+		log_error("Cannot open %s", holders_path);
+		return 0;
+	}
+
+	while ((de = readdir(dr))) {
+		if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, ".."))
+			continue;
+
+		holder_name = de->d_name;
+
+		if (strncmp(holder_name, "dm", 2)) {
+			log_error("Unrecognized holder %s of %s", holder_name, lv_path);
+			ret = 0;
+			break;
+		}
+
+		/* We could read the holder's dm uuid to verify it's a crypt dev. */
+
+		if (dm_snprintf(crypt_path, PATH_MAX, "/dev/%s", holder_name) < 0) {
+			ret = 0;
+			stack;
+			break;
+		}
+		ret = 1;
+		break;
+	}
+	closedir(dr);
+	if (ret)
+		log_debug("Found holder %s of %s.", crypt_path, lv_path);
+	else
+		log_debug("No holder in %s", holders_path);
+	return ret;
+}
+
+int fs_get_info(struct cmd_context *cmd, struct logical_volume *lv,
+		struct fs_info *fsi, int include_mount)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	struct stat st_lv;
+	struct stat st_crypt;
+	struct stat st_top;
+	struct stat stme;
+	struct fs_info info;
+	FILE *fme = NULL;
+	struct mntent *me;
+	int fd;
+	int ret;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir,
+			lv->vg->name, lv->name) < 0) {
+		log_error("Couldn't create LV path for %s.", display_lvname(lv));
+		return 0;
+	}
+
+	if (stat(lv_path, &st_lv) < 0) {
+		log_error("Failed to get LV path %s", lv_path);
+		return 0;
+	}
+
+	memset(&info, 0, sizeof(info));
+
+	if (!fs_get_blkid(lv_path, &info)) {
+		log_error("No file system info from blkid for %s", display_lvname(lv));
+		return 0;
+	}
+
+	if (fsi->nofs)
+		return 1;
+
+	/*
+	 * If there's a LUKS dm-crypt layer over the LV, then
+	 * return fs info from that layer, setting needs_crypt
+	 * to indicate a crypt layer between the fs and LV.
+	 */
+	if (!strcmp(info.fstype, "crypto_LUKS")) {
+		if (!_get_crypt_path(st_lv.st_rdev, lv_path, crypt_path)) {
+			log_error("Cannot find active LUKS dm-crypt device using %s.",
+				  display_lvname(lv));
+			return 0;
+		}
+
+		if (stat(crypt_path, &st_crypt) < 0) {
+			log_error("Failed to get crypt path %s", crypt_path);
+			return 0;
+		}
+
+		memset(&info, 0, sizeof(info));
+
+		log_print("File system found on crypt device %s on LV %s.",
+			  crypt_path, display_lvname(lv));
+
+		if ((fd = open(crypt_path, O_RDONLY)) < 0) {
+			log_error("Failed to open crypt path %s", crypt_path);
+			return 0;
+		}
+		if (ioctl(fd, BLKGETSIZE64, &info.crypt_dev_size_bytes) < 0) {
+			log_error("Failed to get crypt device size %s", crypt_path);
+			close(fd);
+			return 0;
+		}
+		close(fd);
+
+		if (!fs_get_blkid(crypt_path, &info)) {
+			log_error("No file system info from blkid for dm-crypt device %s on LV %s.",
+				  crypt_path, display_lvname(lv));
+			return 0;
+		}
+		*fsi = info;
+		fsi->needs_crypt = 1;
+		fsi->crypt_devt = st_crypt.st_rdev;
+		memcpy(fsi->fs_dev_path, crypt_path, PATH_MAX);
+		st_top = st_crypt;
+
+		if (!get_crypt_table_offset(st_crypt.st_rdev, &fsi->crypt_offset_bytes)) {
+			log_error("Failed to get crypt data offset.");
+			return 0;
+		}
+	} else {
+		*fsi = info;
+		memcpy(fsi->fs_dev_path, lv_path, PATH_MAX);
+		st_top = st_lv;
+	}
+
+	if (!include_mount)
+		return 1;
+
+	if (!(fme = setmntent("/etc/mtab", "r")))
+		return_0;
+
+	ret = 1;
+
+	while ((me = getmntent(fme))) {
+		if (strcmp(me->mnt_type, fsi->fstype))
+			continue;
+		if (me->mnt_dir[0] != '/')
+			continue;
+		if (me->mnt_fsname[0] != '/')
+			continue;
+		if (stat(me->mnt_dir, &stme) < 0)
+			continue;
+		if (stme.st_dev != st_top.st_rdev)
+			continue;
+
+		log_debug("fs_get_info %s is mounted \"%s\"", fsi->fs_dev_path, me->mnt_dir);
+		fsi->mounted = 1;
+		strncpy(fsi->mount_dir, me->mnt_dir, PATH_MAX-1);
+	}
+	endmntent(fme);
+
+	fsi->unmounted = !fsi->mounted;
+	return ret;
+}
+
+#define FS_CMD_MAX_ARGS 16
+
+int crypt_resize_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+			uint64_t newsize_bytes_fs)
+{
+	char crypt_path[PATH_MAX];
+	char newsize_str[16] = { 0 };
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(newsize_str, sizeof(newsize_str), "%llu", (unsigned long long)newsize_bytes_fs) < 0)
+		return_0;
+
+	if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--cryptresize";
+	argv[++args] = "--cryptpath";
+	argv[++args] = crypt_path;
+	argv[++args] = "--newsizebytes";
+	argv[++args] = newsize_str;
+	argv[++args] = NULL;
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to resize crypt dev with lvresize_fs_helper.");
+		return 0;
+	}
+
+	return 1;
+}
+
+/*
+ * The helper script does the following steps for reduce:
+ * devpath = $cryptpath ? $cryptpath : $lvpath
+ * if needs_unmount
+ * 	umount $mountdir 
+ * if needs_fsck
+ * 	e2fsck -f -p $devpath
+ * if needs_mount
+ * 	mount $devpath $tmpdir
+ * if $fstype == "ext"
+ * 	resize2fs $devpath $newsize_kb
+ * if needs_crypt
+ * 	cryptsetup resize --size $newsize_sectors $cryptpath
+ *
+ * Note: when a crypt layer is included, newsize_bytes_fs is smaller
+ * than newsize_bytes_lv because of the crypt header.
+ */
+
+int fs_reduce_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		     uint64_t newsize_bytes_fs, char *fsmode)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	char newsize_str[16] = { 0 };
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	char *devpath;
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(newsize_str, sizeof(newsize_str), "%llu", (unsigned long long)newsize_bytes_fs) < 0)
+		return_0;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir, lv->vg->name, lv->name) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--fsreduce";
+	argv[++args] = "--fstype";
+	argv[++args] = fsi->fstype;
+	argv[++args] = "--lvpath";
+	argv[++args] = lv_path;
+
+	if (newsize_bytes_fs) {
+		argv[++args] = "--newsizebytes";
+		argv[++args] = newsize_str;
+	}
+	if (fsi->mounted) {
+		argv[++args] = "--mountdir";
+		argv[++args] = fsi->mount_dir;
+	}
+
+	if (fsi->needs_unmount)
+		argv[++args] = "--unmount";
+	if (fsi->needs_mount)
+		argv[++args] = "--mount";
+	if (fsi->needs_fsck)
+		argv[++args] = "--fsck";
+
+	if (fsi->needs_crypt) {
+		if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+			return_0;
+		argv[++args] = "--cryptresize";
+		argv[++args] = "--cryptpath";
+		argv[++args] = crypt_path;
+	}
+
+	/*
+	 * fsmode manage means the fs should be remounted after
+	 * resizing if it was unmounted.
+	 */
+	if (fsi->needs_unmount && !strcmp(fsmode, "manage"))
+		argv[++args] = "--remount";
+
+	argv[++args] = NULL;
+
+	devpath = fsi->needs_crypt ? crypt_path : (char *)display_lvname(lv);
+
+	log_print("Reducing file system %s to %s (%llu bytes) on %s...",
+		  fsi->fstype, display_size(cmd, newsize_bytes_fs/512),
+		  (unsigned long long)newsize_bytes_fs, devpath);
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to reduce file system with lvresize_fs_helper.");
+		return 0;
+	}
+
+	log_print("Reduced file system %s on %s.", fsi->fstype, devpath);
+
+	return 1;
+}
+
+/*
+ * The helper script does the following steps for extend:
+ * devpath = $cryptpath ? $cryptpath : $lvpath
+ * if needs_unmount
+ * 	umount $mountdir 
+ * if needs_fsck
+ * 	e2fsck -f -p $devpath
+ * if needs_crypt
+ * 	cryptsetup resize $cryptpath
+ * if needs_mount
+ * 	mount $devpath $tmpdir
+ * if $fstype == "ext"
+ * 	resize2fs $devpath
+ * if $fstype == "xfs"
+ * 	xfs_growfs $devpath
+ *
+ * Note: when a crypt layer is included, newsize_bytes_fs is smaller
+ * than newsize_bytes_lv because of the crypt header.
+ */
+
+int fs_extend_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		     uint64_t newsize_bytes_fs, char *fsmode)
+{
+	char lv_path[PATH_MAX];
+	char crypt_path[PATH_MAX];
+	const char *argv[FS_CMD_MAX_ARGS + 4];
+	char *devpath;
+	int args = 0;
+	int status;
+
+	if (dm_snprintf(lv_path, PATH_MAX, "%s%s/%s", lv->vg->cmd->dev_dir, lv->vg->name, lv->name) < 0)
+		return_0;
+
+	argv[0] = _get_lvresize_fs_helper_path();
+	argv[++args] = "--fsextend";
+	argv[++args] = "--fstype";
+	argv[++args] = fsi->fstype;
+	argv[++args] = "--lvpath";
+	argv[++args] = lv_path;
+
+	if (fsi->mounted) {
+		argv[++args] = "--mountdir";
+		argv[++args] = fsi->mount_dir;
+	}
+
+	if (fsi->needs_unmount)
+		argv[++args] = "--unmount";
+	if (fsi->needs_mount)
+		argv[++args] = "--mount";
+	if (fsi->needs_fsck)
+		argv[++args] = "--fsck";
+
+	if (fsi->needs_crypt) {
+		if (dm_snprintf(crypt_path, sizeof(crypt_path), "/dev/dm-%d", (int)MINOR(fsi->crypt_devt)) < 0)
+			return_0;
+		argv[++args] = "--cryptresize";
+		argv[++args] = "--cryptpath";
+		argv[++args] = crypt_path;
+	}
+
+	/*
+	 * fsmode manage means the fs should be remounted after
+	 * resizing if it was unmounted.
+	 */
+	if (fsi->needs_unmount && !strcmp(fsmode, "manage"))
+		argv[++args] = "--remount";
+
+	argv[++args] = NULL;
+
+	devpath = fsi->needs_crypt ? crypt_path : (char *)display_lvname(lv);
+
+	log_print("Extending file system %s to %s (%llu bytes) on %s...",
+		  fsi->fstype, display_size(cmd, newsize_bytes_fs/512),
+		  (unsigned long long)newsize_bytes_fs, devpath);
+
+	if (!exec_cmd(cmd, argv, &status, 1)) {
+		log_error("Failed to extend file system with lvresize_fs_helper.");
+		return 0;
+	}
+
+	log_print("Extended file system %s on %s.", fsi->fstype, devpath);
+
+	return 1;
+}
+

lib/device/filesystem.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _FILESYSTEM_H
+#define _FILESYSTEM_H
+
+#define FSTYPE_MAX 16
+
+struct fs_info {
+	char fstype[FSTYPE_MAX];
+	char mount_dir[PATH_MAX];
+	char fs_dev_path[PATH_MAX]; /* usually lv dev, can be crypt dev */
+	unsigned int fs_block_size_bytes; /* 512 or 4k */
+	uint64_t fs_last_byte; /* last byte on the device used by the fs */
+	uint32_t crypt_offset_bytes; /* offset in bytes of crypt data on LV */
+	dev_t crypt_devt; /* dm-crypt device between the LV and FS */
+	uint64_t crypt_dev_size_bytes;
+
+	unsigned nofs:1;
+	unsigned unmounted:1;
+	unsigned mounted:1;
+	/* for resizing */
+	unsigned needs_reduce:1;
+	unsigned needs_extend:1;
+	unsigned needs_fsck:1;
+	unsigned needs_unmount:1;
+	unsigned needs_mount:1;
+	unsigned needs_crypt:1;
+};
+
+int fs_get_info(struct cmd_context *cmd, struct logical_volume *lv,
+                struct fs_info *fsi, int include_mount);
+
+int fs_extend_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes, char *fsmode);
+int fs_reduce_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes, char *fsmode);
+int crypt_resize_script(struct cmd_context *cmd, struct logical_volume *lv, struct fs_info *fsi,
+		uint64_t newsize_bytes_fs);
+#endif

lib/device/online.c

@@ -16,7 +16,6 @@
 #include "lib/misc/lib.h"
 #include "lib/commands/toolcontext.h"
 #include "lib/device/device.h"
-#include "lib/device/device_id.h"
 #include "lib/device/online.h"
 
 #include <dirent.h>
@@ -505,3 +504,58 @@ do_lookup:
 	if ((rv < 0) && stat(PVS_LOOKUP_DIR, &st))
 		log_error_pvscan(cmd, "Failed to create %s %d", PVS_LOOKUP_DIR, errno);
 }
+
+void online_lookup_file_remove(const char *vgname)
+{
+	char path[PATH_MAX];
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_LOOKUP_DIR, vgname) < 0) {
+		log_error("Path %s/%s is too long.", PVS_LOOKUP_DIR, vgname);
+		return;
+	}
+
+	log_debug("Unlink pvs_lookup: %s", path);
+
+	if (unlink(path) && (errno != ENOENT))
+		log_sys_debug("unlink", path);
+}
+
+static int _online_pvid_file_remove(char *pvid)
+{
+	char path[PATH_MAX];
+
+	if (dm_snprintf(path, sizeof(path), "%s/%s", PVS_ONLINE_DIR, pvid) < 0)
+		return_0;
+	if (!unlink(path))
+		return 1;
+	return 0;
+}
+
+/*
+ * Reboot automatically clearing tmpfs on /run is the main method of removing
+ * online files.  It's important to note that removing the online files for a
+ * VG is not a technical requirement for anything and could easily be skipped
+ * if it had any downside.  It's only done to clean up the space used in /run
+ * by the online files, e.g. if there happens to be an extreme amount of
+ * vgcreate/pvscan/vgremove between reboots that are leaving a large number of
+ * useless online files consuming tmpfs space.
+ */
+void online_vgremove(struct volume_group *vg)
+{
+	char pvid[ID_LEN + 1] __attribute__((aligned(8))) = { 0 };
+	struct pv_list *pvl;
+
+	/*
+	 * online files may not exist for the vg if there has been no
+	 * pvscans or autoactivation.
+	 */
+
+	online_vg_file_remove(vg->name);
+	online_lookup_file_remove(vg->name);
+
+	dm_list_iterate_items(pvl, &vg->pvs) {
+		memcpy(pvid, &pvl->pv->id.uuid, ID_LEN);
+		_online_pvid_file_remove(pvid);
+	}
+}
+

lib/device/online.h

@@ -54,5 +54,7 @@ void online_dir_setup(struct cmd_context *cmd);
 int get_pvs_online(struct dm_list *pvs_online, const char *vgname);
 int get_pvs_lookup(struct dm_list *pvs_online, const char *vgname);
 void free_po_list(struct dm_list *list);
+void online_lookup_file_remove(const char *vgname);
+void online_vgremove(struct volume_group *vg);
 
 #endif

lib/device/parse_vpd.c

@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2022 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/device/device.h"
+#include "lib/device/device_id.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <limits.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <assert.h>
+
+/*
+ * Replace series of spaces with a single _.
+ */
+int format_t10_id(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int in_space = 0;
+	int retlen = 0;
+	int j = 0;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		if (!in[i])
+			break;
+		if (j >= (out_bytes - 2))
+			break;
+		/* skip leading spaces */
+		if (!retlen && (in[i] == ' '))
+			continue;
+		/* replace one or more spaces with _ */
+		if (in[i] == ' ') {
+			in_space = 1;
+			continue;
+		}
+		/* spaces are finished so insert _ */
+		if (in_space) {
+			out[j++] = '_';
+			in_space = 0;
+			retlen++;
+		}
+		out[j++] = in[i];
+		retlen++;
+	}
+	return retlen;
+}
+
+static int _to_hex(const unsigned char *in, int in_bytes, unsigned char *out, int out_bytes)
+{
+	int off = 0;
+	int num;
+	int i;
+
+	for (i = 0; i < in_bytes; i++) {
+		num = sprintf((char *)out + off, "%02x", in[i]);
+		if (num < 0)
+			break;
+		off += num;
+		if (off + 2 >= out_bytes)
+			break;
+	}
+	return off;
+}
+
+#define ID_BUFSIZE 1024
+
+/*
+ * based on linux kernel function
+ */
+int parse_vpd_ids(const unsigned char *vpd_data, int vpd_datalen, struct dm_list *ids)
+{
+	char id[ID_BUFSIZE];
+	unsigned char tmp_str[ID_BUFSIZE];
+	const unsigned char *d, *cur_id_str;
+	size_t id_len = ID_BUFSIZE;
+	int id_size = -1;
+	int type;
+	uint8_t cur_id_size = 0;
+
+	memset(id, 0, ID_BUFSIZE);
+	for (d = vpd_data + 4;
+	     d < vpd_data + vpd_datalen;
+	     d += d[3] + 4) {
+		memset(tmp_str, 0, sizeof(tmp_str));
+
+		switch (d[1] & 0xf) {
+		case 0x1:
+			/* T10 Vendor ID */
+			cur_id_size = d[3];
+			if (cur_id_size + 4 > id_len)
+				cur_id_size = id_len - 4;
+			cur_id_str = d + 4;
+			format_t10_id(cur_id_str, cur_id_size, tmp_str, sizeof(tmp_str));
+			id_size = snprintf(id, ID_BUFSIZE, "t10.%s", tmp_str);
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 1, ids);
+			break;
+		case 0x2:
+			/* EUI-64 */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 12:
+				_to_hex(cur_id_str, 12, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "eui.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 2, ids);
+			break;
+		case 0x3:
+			/* NAA */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			switch (cur_id_size) {
+			case 8:
+				_to_hex(cur_id_str, 8, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			case 16:
+				_to_hex(cur_id_str, 16, tmp_str, sizeof(tmp_str));
+				id_size = snprintf(id, ID_BUFSIZE, "naa.%s", tmp_str);
+				break;
+			default:
+				break;
+			}
+			if (id_size < 0)
+				break;
+			if (id_size >= ID_BUFSIZE)
+				id_size = ID_BUFSIZE - 1;
+			dev_add_wwid(id, 3, ids);
+			break;
+		case 0x8:
+			/* SCSI name string */
+			cur_id_size = d[3];
+			cur_id_str = d + 4;
+			if (cur_id_size >= id_len)
+				cur_id_size = id_len - 1;
+			memcpy(id, cur_id_str, cur_id_size);
+			id_size = cur_id_size;
+
+			/*
+			 * if naa or eui ids are provided as scsi names,
+			 * consider them to be naa/eui types.
+			 */
+			if (!memcmp(id, "eui.", 4))
+				type = 2;
+			else if (!memcmp(id, "naa.", 4))
+				type = 3;
+			else
+				type = 8;
+
+			/*
+			 * Not in the kernel version, copying multipath code,
+			 * which checks if this string begins with naa or eui
+			 * and if so does tolower() on the chars.
+			 */
+			if ((type == 2) || (type == 3)) {
+				int i;
+				for (i = 0; i < strlen(id); i++)
+					id[i] = tolower(id[i]);
+			}
+			dev_add_wwid(id, type, ids);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return id_size;
+}
+
+int parse_vpd_serial(const unsigned char *in, char *out, int outsize)
+{
+	uint8_t len_buf[2] __attribute__((aligned(8))) = { 0 };;
+	size_t len;
+
+	/* parsing code from multipath tools */
+	/* ignore in[0] and in[1] */
+	/* len is in[2] and in[3] */
+	/* serial begins at in[4] */
+
+	len_buf[0] = in[2];
+	len_buf[1] = in[3];
+	len = len_buf[0] << 8 | len_buf[1];
+
+	if (outsize == 0)
+		return 0;
+
+	if (len > DEV_WWID_SIZE)
+		len = DEV_WWID_SIZE;
+	/*
+	 * Strip leading and trailing whitespace
+	 */
+	while (len > 0 && in[len + 3] == ' ')
+		--len;
+	while (len > 0 && in[4] == ' ') {
+		++in;
+		--len;
+	}
+
+	if (len >= outsize)
+                len = outsize - 1;
+
+	if (len > 0) {
+		memcpy(out, in + 4, len);
+		out[len] = '\0';
+	}
+	return len;
+}
+

lib/filters/filter-internal.c

@@ -1,86 +0,0 @@
-/*
- * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
- *
- * This file is part of LVM2.
- *
- * This copyrighted material is made available to anyone wishing to use,
- * modify, copy, or redistribute it subject to the terms and conditions
- * of the GNU Lesser General Public License v.2.1.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include "base/memory/zalloc.h"
-#include "lib/misc/lib.h"
-#include "lib/filters/filter.h"
-
-static DM_LIST_INIT(_allow_devs);
-
-int internal_filter_allow(struct dm_pool *mem, struct device *dev)
-{
-	struct device_list *devl;
-
-	if (!(devl = dm_pool_alloc(mem, sizeof(*devl)))) {
-		log_error("device_list element allocation failed");
-		return 0;
-	}
-	devl->dev = dev;
-
-	dm_list_add(&_allow_devs, &devl->list);
-	return 1;
-}
-
-void internal_filter_clear(void)
-{
-	dm_list_init(&_allow_devs);
-}
-
-static int _passes_internal(struct cmd_context *cmd, struct dev_filter *f __attribute__((unused)),
-			    struct device *dev, const char *use_filter_name)
-{
-	struct device_list *devl;
-
-	dev->filtered_flags &= ~DEV_FILTERED_INTERNAL;
-
-	if (!internal_filtering())
-		return 1;
-	
-	dm_list_iterate_items(devl, &_allow_devs) {
-		if (devl->dev == dev)
-			return 1;
-	}
-	
-	dev->filtered_flags |= DEV_FILTERED_INTERNAL;
-	log_debug_devs("%s: Skipping for internal filtering.", dev_name(dev));
-	return 0;
-}
-
-static void _destroy(struct dev_filter *f)
-{
-	if (f->use_count)
-		log_error(INTERNAL_ERROR "Destroying internal filter while in use %u times.", f->use_count);
-
-	free(f);
-}
-
-struct dev_filter *internal_filter_create(void)
-{
-	struct dev_filter *f;
-
-	if (!(f = zalloc(sizeof(*f)))) {
-		log_error("md filter allocation failed");
-		return NULL;
-	}
-
-	f->passes_filter = _passes_internal;
-	f->destroy = _destroy;
-	f->use_count = 0;
-	f->name = "internal";
-
-	log_debug_devs("Internal filter initialised.");
-
-	return f;
-}
-

lib/filters/filter-md.c

@@ -99,14 +99,6 @@ static int _passes_md_filter(struct cmd_context *cmd, struct dev_filter *f __att
 		return 1;
 
 	ret = dev_is_md_component(cmd, dev, NULL, cmd->use_full_md_check);
-
-	if (ret == -EAGAIN) {
-		/* let pass, call again after scan */
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		log_debug_devs("filter md deferred %s", dev_name(dev));
-		return 1;
-	}
-
 	if (ret == 0)
 		return 1;
 

lib/filters/filter-mpath.c

@@ -15,18 +15,41 @@
 #include "base/memory/zalloc.h"
 #include "lib/misc/lib.h"
 #include "lib/filters/filter.h"
+#include "lib/device/device_id.h"
 
 #ifdef __linux__
 
 #include <dirent.h>
 
+static int _lvmdevices_update_msg;
+
 static int _ignore_mpath_component(struct cmd_context *cmd, struct dev_filter *f, struct device *dev, const char *use_filter_name)
 {
+	dev_t mpath_devno = 0;
+
 	dev->filtered_flags &= ~DEV_FILTERED_MPATH_COMPONENT;
 
-	if (dev_is_mpath_component(cmd, dev)) {
+	if (dev_is_mpath_component(cmd, dev, &mpath_devno)) {
 		log_debug_devs("%s: Skipping mpath component device", dev_name(dev));
 		dev->filtered_flags |= DEV_FILTERED_MPATH_COMPONENT;
+
+		/*
+		 * Warn about misconfig where an mpath component is
+		 * in the devices file, but its mpath device is not.
+		 */
+		if ((dev->flags & DEV_MATCHED_USE_ID) && mpath_devno) {
+			if (!get_du_for_devno(cmd, mpath_devno)) {
+				struct device *mpath_dev = dev_cache_get_by_devt(cmd, mpath_devno);
+				log_warn("WARNING: devices file is missing %s (%d:%d) using multipath component %s.",
+					 mpath_dev ? dev_name(mpath_dev) : "unknown",
+					 (int)MAJOR(mpath_devno), (int)MINOR(mpath_devno), dev_name(dev));
+				if (!_lvmdevices_update_msg && strcmp(get_cmd_name(), "lvmdevices")) {
+					log_warn("See lvmdevices --update for devices file update.");
+					_lvmdevices_update_msg = 1;
+				}
+			}
+		}
+
 		return 0;
 	}
 

lib/filters/filter-partitioned.c

@@ -30,14 +30,6 @@ static int _passes_partitioned_filter(struct cmd_context *cmd, struct dev_filter
 	dev->filtered_flags &= ~DEV_FILTERED_PARTITIONED;
 
 	ret = dev_is_partitioned(cmd, dev);
-
-	if (ret == -EAGAIN) {
-		/* let pass, call again after scan */
-		log_debug_devs("filter partitioned deferred %s", dev_name(dev));
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		return 1;
-	}
-
 	if (ret) {
 		if (dev->ext.src == DEV_EXT_NONE)
 			log_debug_devs(MSG_SKIPPING, dev_name(dev));

lib/filters/filter-persistent.c

@@ -109,8 +109,6 @@ static int _lookup_p(struct cmd_context *cmd, struct dev_filter *f, struct devic
 
 	/* Uncached, check filters and cache the result */
 	if (!l) {
-		dev->flags &= ~DEV_FILTER_AFTER_SCAN;
-
 		pass = pf->real->passes_filter(cmd, pf->real, dev, use_filter_name);
 
 		if (!pass) {
@@ -120,21 +118,13 @@ static int _lookup_p(struct cmd_context *cmd, struct dev_filter *f, struct devic
 			 * because the deferred result won't change the exclude.
 			 */
 			l = PF_BAD_DEVICE;
-
-		} else if ((pass == -EAGAIN) || (dev->flags & DEV_FILTER_AFTER_SCAN)) {
-			/*
-			 * When the filter result is deferred, we let the device
-			 * pass for now, but do not cache the result.  We need to
-			 * rerun the filters later.  At that point the final result
-			 * will be cached.
-			 */
-			log_debug_devs("filter cache deferred %s", dev_name(dev));
-			dev->flags |= DEV_FILTER_AFTER_SCAN;
-			pass = 1;
-			goto out;
-
-		} else if (pass) {
+		} else if (pass == 1) {
 			l = PF_GOOD_DEVICE;
+		} else {
+			log_error("Ignore invalid filter result %d %s", pass, dev_name(dev));
+			pass = 1;
+			/* don't cache invalid result */
+			goto out;
 		}
 
 		if (!dev->filtered_flags) /* skipping reason already logged by filter */

lib/filters/filter-signature.c

@@ -33,13 +33,6 @@ static int _ignore_signature(struct cmd_context *cmd, struct dev_filter *f __att
 
 	dev->filtered_flags &= ~DEV_FILTERED_SIGNATURE;
 
-	if (!scan_bcache) {
-		/* let pass, call again after scan */
-		log_debug_devs("filter signature deferred %s", dev_name(dev));
-		dev->flags |= DEV_FILTER_AFTER_SCAN;
-		return 1;
-	}
-
 	memset(buf, 0, BUFSIZE);
 
 	if (!dev_read_bytes(dev, 0, BUFSIZE, buf)) {

lib/filters/filter.h

@@ -32,10 +32,6 @@ struct dev_filter *sysfs_filter_create(void);
 struct dev_filter *signature_filter_create(struct dev_types *dt);
 struct dev_filter *deviceid_filter_create(struct cmd_context *cmd);
 
-struct dev_filter *internal_filter_create(void);
-int internal_filter_allow(struct dm_pool *mem, struct device *dev);
-void internal_filter_clear(void);
-
 /*
  * patterns must be an array of strings of the form:
  * [ra]<sep><regex><sep>, eg,

lib/format_text/format-text.c

@@ -428,7 +428,7 @@ static struct volume_group *_vg_read_raw_area(struct cmd_context *cmd,
 				rlocn->checksum,
 				&when, &desc);
 
-	if (!vg && !*use_previous_vg) {
+	if (!vg && (!use_previous_vg || !*use_previous_vg)) {
 		log_warn("WARNING: Failed to read metadata text at %llu off %llu size %llu VG %s on %s",
 			 (unsigned long long)(area->start + rlocn->offset),
 			 (unsigned long long)rlocn->offset,

lib/label/hints.c

@@ -146,7 +146,6 @@
 #include "lib/label/hints.h"
 #include "lib/device/dev-type.h"
 #include "lib/device/device_id.h"
-#include "lib/device/online.h"
 
 #include <sys/stat.h>
 #include <fcntl.h>

lib/label/label.c

@@ -459,7 +459,6 @@ static int _scan_dev_open(struct device *dev)
 	const char *name;
 	const char *modestr;
 	struct stat sbuf;
-	int retried = 0;
 	int flags = 0;
 	int fd, di;
 
@@ -479,14 +478,23 @@ static int _scan_dev_open(struct device *dev)
 		return 0;
 	}
 
+ next_name:
 	/*
 	 * All the names for this device (major:minor) are kept on
 	 * dev->aliases, the first one is the primary/preferred name.
+	 *
+	 * The default name preferences in dev-cache mean that the first
+	 * name in dev->aliases is not a symlink for scsi devices, but is
+	 * the /dev/mapper/ symlink for mpath devices.
+	 *
+	 * If preferred names are set to symlinks, should this
+	 * first attempt to open using a non-symlink?
+	 *
+	 * dm_list_first() returns NULL if the list is empty.
 	 */
 	if (!(name_list = dm_list_first(&dev->aliases))) {
-		/* Shouldn't happen */
-		log_error("Device open %s %d:%d has no path names.",
-			  dev_name(dev), (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		log_error("Device open %d:%d has no path names.",
+			  (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
 		return 0;
 	}
 	name_sl = dm_list_item(name_list, struct dm_str_list);
@@ -514,50 +522,34 @@ static int _scan_dev_open(struct device *dev)
 		modestr = "ro";
 	}
 
-retry_open:
-
 	fd = open(name, flags, 0777);
-
 	if (fd < 0) {
 		if ((errno == EBUSY) && (flags & O_EXCL)) {
 			log_error("Can't open %s exclusively.  Mounted filesystem?",
 				  dev_name(dev));
+			return 0;
 		} else {
-			int major, minor;
-
 			/*
-			 * Shouldn't happen, if it does, print stat info to help figure
-			 * out what's wrong.
+			 * drop name from dev->aliases and use verify_aliases to
+			 * drop any other invalid aliases before retrying open with
+			 * any remaining valid paths.
 			 */
-
-			major = (int)MAJOR(dev->dev);
-			minor = (int)MINOR(dev->dev);
-
-			log_error("Device open %s %d:%d failed errno %d", name, major, minor, errno);
-
-			if (stat(name, &sbuf)) {
-				log_debug_devs("Device open %s %d:%d stat failed errno %d",
-					       name, major, minor, errno);
-			} else if (sbuf.st_rdev != dev->dev) {
-				log_debug_devs("Device open %s %d:%d stat %d:%d does not match.",
-					       name, major, minor,
-					       (int)MAJOR(sbuf.st_rdev), (int)MINOR(sbuf.st_rdev));
-			}
-
-			if (!retried) {
-				/*
-				 * FIXME: remove this, the theory for this retry is that
-				 * there may be a udev race that we can sometimes mask by
-				 * retrying.  This is here until we can figure out if it's
-				 * needed and if so fix the real problem.
-				 */
-				usleep(5000);
-				log_debug_devs("Device open %s retry", dev_name(dev));
-				retried = 1;
-				goto retry_open;
-			}
+			log_debug("Drop alias for %d:%d failed open %s (%d)",
+				  (int)MAJOR(dev->dev), (int)MINOR(dev->dev), name, errno);
+			dev_cache_failed_path(dev, name);
+			dev_cache_verify_aliases(dev);
+			goto next_name;
 		}
-		return 0;
+	}
+
+	/* Verify that major:minor from the path still match dev. */
+	if ((fstat(fd, &sbuf) < 0) || (sbuf.st_rdev != dev->dev)) {
+		log_warn("Invalid path %s for device %d:%d, trying different path.",
+			 name, (int)MAJOR(dev->dev), (int)MINOR(dev->dev));
+		(void)close(fd);
+		dev_cache_failed_path(dev, name);
+		dev_cache_verify_aliases(dev);
+		goto next_name;
 	}
 
 	dev->flags |= DEV_IN_BCACHE;
@@ -605,37 +597,6 @@ static int _scan_dev_close(struct device *dev)
 	return 1;
 }
 
-static void _drop_bad_aliases(struct device *dev)
-{
-	struct dm_str_list *strl, *strl2;
-	const char *name;
-	struct stat sbuf;
-	int major = (int)MAJOR(dev->dev);
-	int minor = (int)MINOR(dev->dev);
-	int bad;
-
-	dm_list_iterate_items_safe(strl, strl2, &dev->aliases) {
-		name = strl->str;
-		bad = 0;
-
-		if (stat(name, &sbuf)) {
-			bad = 1;
-			log_debug_devs("Device path check %d:%d %s stat failed errno %d",
-					major, minor, name, errno);
-		} else if (sbuf.st_rdev != dev->dev) {
-			bad = 1;
-			log_debug_devs("Device path check %d:%d %s stat %d:%d does not match.",
-				       major, minor, name,
-				       (int)MAJOR(sbuf.st_rdev), (int)MINOR(sbuf.st_rdev));
-		}
-
-		if (bad) {
-			log_debug_devs("Device path check %d:%d dropping path %s.", major, minor, name);
-			dev_cache_failed_path(dev, name);
-		}
-	}
-}
-
 // Like bcache_invalidate, only it throws any dirty data away if the
 // write fails.
 static void _invalidate_di(struct bcache *cache, int di)
@@ -663,10 +624,8 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 	char headers_buf[HEADERS_BUF_SIZE];
 	struct dm_list wait_devs;
 	struct dm_list done_devs;
-	struct dm_list reopen_devs;
 	struct device_list *devl, *devl2;
 	struct block *bb;
-	int retried_open = 0;
 	int scan_read_errors = 0;
 	int scan_process_errors = 0;
 	int scan_failed_count = 0;
@@ -677,7 +636,6 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 
 	dm_list_init(&wait_devs);
 	dm_list_init(&done_devs);
-	dm_list_init(&reopen_devs);
 
 	log_debug_devs("Scanning %d devices for VG info", dm_list_size(devs));
 
@@ -701,9 +659,9 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 
 		if (!_in_bcache(devl->dev)) {
 			if (!_scan_dev_open(devl->dev)) {
-				log_debug_devs("Scan failed to open %s.", dev_name(devl->dev));
+				log_debug_devs("Scan failed to open %d:%d %s.",
+					       (int)MAJOR(devl->dev->dev), (int)MINOR(devl->dev->dev), dev_name(devl->dev));
 				dm_list_del(&devl->list);
-				dm_list_add(&reopen_devs, &devl->list);
 				devl->dev->flags |= DEV_SCAN_NOT_READ;
 				continue;
 			}
@@ -787,41 +745,6 @@ static int _scan_list(struct cmd_context *cmd, struct dev_filter *f,
 	if (!dm_list_empty(devs))
 		goto scan_more;
 
-	/*
-	 * We're done scanning all the devs.  If we failed to open any of them
-	 * the first time through, refresh device paths and retry.  We failed
-	 * to open the devs on the reopen_devs list.
-	 *
-	 * FIXME: it's not clear if or why this helps.
-	 */
-	if (!dm_list_empty(&reopen_devs)) {
-		if (retried_open) {
-			/* Don't try again. */
-			scan_failed_count += dm_list_size(&reopen_devs);
-			dm_list_splice(&done_devs, &reopen_devs);
-			goto out;
-		}
-		retried_open = 1;
-
-		dm_list_iterate_items_safe(devl, devl2, &reopen_devs) {
-			_drop_bad_aliases(devl->dev);
-
-			if (dm_list_empty(&devl->dev->aliases)) {
-				log_warn("WARNING: Scan ignoring device %d:%d with no paths.",
-					 (int)MAJOR(devl->dev->dev),
-					 (int)MINOR(devl->dev->dev));
-					 
-				dm_list_del(&devl->list);
-				lvmcache_del_dev(devl->dev);
-				scan_failed_count++;
-			}
-		}
-
-		/* Put devs that failed to open back on the original list to retry. */
-		dm_list_splice(devs, &reopen_devs);
-		goto scan_more;
-	}
-out:
 	log_debug_devs("Scanned devices: read errors %d process errors %d failed %d",
 			scan_read_errors, scan_process_errors, scan_failed_count);
 
@@ -878,7 +801,7 @@ static int _setup_bcache(void)
 	}
 
 	if (!(scan_bcache = bcache_create(BCACHE_BLOCK_SIZE_IN_SECTORS, cache_blocks, ioe))) {
-		log_error("Failed to create bcache with %d cache blocks.", cache_blocks);
+		log_error("Failed to set up io layer with %d blocks.", cache_blocks);
 		return 0;
 	}
 
@@ -1180,7 +1103,7 @@ int label_scan_vg_online(struct cmd_context *cmd, const char *vgname,
 
 	dm_list_iterate_items_safe(devl, devl2, &devs) {
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, devl->dev, NULL)) {
-			log_print("%s excluded by filters: %s.",
+			log_print("%s excluded: %s.",
 				  dev_name(devl->dev), dev_filtered_reason(devl->dev));
 			dm_list_del(&devl->list);
 			dm_list_add(&devs_drop, &devl->list);
@@ -1246,7 +1169,7 @@ int label_scan_vg_online(struct cmd_context *cmd, const char *vgname,
 
 		/* Applies all filters, including those that need data from dev. */
 		if (!cmd->filter->passes_filter(cmd, cmd->filter, devl->dev, NULL)) {
-			log_print("%s excluded by filters: %s.",
+			log_print("%s excluded: %s.",
 				  dev_name(devl->dev), dev_filtered_reason(devl->dev));
 			dm_list_del(&devl->list);
 			dm_list_add(&devs_drop, &devl->list);
@@ -1369,7 +1292,7 @@ int label_scan(struct cmd_context *cmd)
 	 * data to invalidate.)
 	 */
 	if (!(iter = dev_iter_create(NULL, 0))) {
-		log_error("Scanning failed to get devices.");
+		log_error("Failed to get device list.");
 		return 0;
 	}
 	while ((dev = dev_iter_get(cmd, iter))) {
@@ -1738,9 +1661,11 @@ void label_scan_invalidate_lvs(struct cmd_context *cmd, struct dm_list *lvs)
 	dev_t devt;
 
 	/*
-	 * FIXME: this is all unnecessary unless there are PVs stacked on LVs,
-	 * so we can skip all of this if scan_lvs=0.
+	 * This is only needed when the command sees PVs stacked on LVs which
+	 * will only happen with scan_lvs=1.
 	 */
+	if (!cmd->scan_lvs)
+		return;
 	log_debug("invalidating devs for any pvs on lvs");
 
 	if (get_device_list(NULL, &devs, &devs_features)) {

lib/locking/lvmlockd.c

@@ -513,9 +513,11 @@ static int _create_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
 
 	lv_size_bytes = num_mb * ONE_MB_IN_BYTES;  /* size of sanlock LV in bytes */
 	extent_bytes = vg->extent_size * SECTOR_SIZE; /* size of one extent in bytes */
-	total_extents = lv_size_bytes / extent_bytes; /* number of extents in sanlock LV */
+	total_extents = dm_div_up(lv_size_bytes, extent_bytes); /* number of extents in sanlock LV */
 	lp.extents = total_extents;
 
+	lv_size_bytes = total_extents * extent_bytes;
+	num_mb = lv_size_bytes / ONE_MB_IN_BYTES;
 	log_debug("Creating lvmlock LV for sanlock with size %um %ub %u extents", num_mb, lv_size_bytes, lp.extents);
 
 	dm_list_init(&lp.tags);
@@ -577,8 +579,9 @@ static int _extend_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg,
 		  (uint32_t)(new_size_sectors * SECTOR_SIZE));
 
 	lp.size = new_size_sectors;
+	lp.pvh = &vg->pvs;
 
-	if (!lv_resize(lv, &lp, &vg->pvs)) {
+	if (!lv_resize(cmd, lv, &lp)) {
 		log_error("Extend sanlock LV %s to size %s failed.",
 			  display_lvname(lv), display_size(cmd, lp.size));
 		return 0;
@@ -2295,12 +2298,11 @@ int lockd_vg_update(struct volume_group *vg)
 	return ret;
 }
 
-static int _query_lock_lv(struct cmd_context *cmd, struct volume_group *vg,
-			  const char *lv_name, char *lv_uuid,
-			  const char *lock_args, int *ex, int *sh)
+static int _query_lv(struct cmd_context *cmd, struct volume_group *vg,
+		     const char *lv_name, char *lv_uuid, const char *lock_args,
+		     int *ex, int *sh)
 {
 	daemon_reply reply;
-	const char *opts = NULL;
 	const char *reply_str;
 	int result;
 	int ret;
@@ -2309,7 +2311,7 @@ static int _query_lock_lv(struct cmd_context *cmd, struct volume_group *vg,
 
 	reply = _lockd_send("query_lock_lv",
 				"pid = " FMTd64, (int64_t) getpid(),
-				"opts = %s", opts ?: "none",
+				"opts = %s", "none",
 				"vg_name = %s", vg->name,
 				"lv_name = %s", lv_name,
 				"lv_uuid = %s", lv_uuid,
@@ -2345,6 +2347,37 @@ static int _query_lock_lv(struct cmd_context *cmd, struct volume_group *vg,
 	return ret;
 }
 
+int lockd_query_lv(struct cmd_context *cmd, struct logical_volume *lv, int *ex, int *sh)
+{
+	struct volume_group *vg = lv->vg;
+	char lv_uuid[64] __attribute__((aligned(8)));
+
+	if (cmd->lockd_lv_disable)
+		return 1;
+	if (!vg_is_shared(vg))
+		return 1;
+	if (!_use_lvmlockd)
+		return 0;
+	if (!_lvmlockd_connected)
+		return 0;
+
+	/* types that cannot be active concurrently will always be ex. */
+	if (lv_is_external_origin(lv) ||
+	    lv_is_thin_type(lv) ||
+	    lv_is_mirror_type(lv) ||
+	    lv_is_raid_type(lv) ||
+	    lv_is_vdo_type(lv) ||
+	    lv_is_cache_type(lv)) {
+		*ex = 1;
+		return 1;
+	}
+
+	if (!id_write_format(&lv->lvid.id[1], lv_uuid, sizeof(lv_uuid)))
+		return_0;
+
+	return _query_lv(cmd, vg, lv->name, lv_uuid, lv->lock_args, ex, sh);
+}
+
 /*
  * When this is called directly (as opposed to being called from
  * lockd_lv), the caller knows that the LV has a lock.
@@ -2383,7 +2416,7 @@ int lockd_lv_name(struct cmd_context *cmd, struct volume_group *vg,
 		    !strcmp(cmd->name, "lvchange") || !strcmp(cmd->name, "lvconvert")) {
 			int ex = 0, sh = 0;
 
-			if (!_query_lock_lv(cmd, vg, lv_name, lv_uuid, lock_args, &ex, &sh))
+			if (!_query_lv(cmd, vg, lv_name, lv_uuid, lock_args, &ex, &sh))
 				return 1;
 			if (sh) {
 				log_warn("WARNING: shared LV may require refresh on other hosts where it is active.");
@@ -2555,7 +2588,7 @@ static int _lockd_lv_thin(struct cmd_context *cmd, struct logical_volume *lv,
 	 * Unlock when the pool is no longer active.
 	 */
 
-	if (def_mode && !strcmp(def_mode, "un") && pool_is_active(pool_lv))
+	if (def_mode && !strcmp(def_mode, "un") && thin_pool_is_active(pool_lv))
 		return 1;
 
 	flags |= LDLV_MODE_NO_SH;
@@ -2733,6 +2766,9 @@ int lockd_lv_resize(struct cmd_context *cmd, struct logical_volume *lv,
 	if (!_lvmlockd_connected)
 		return 0;
 
+	if (lv_is_lockd_sanlock_lv(lv))
+		return 1;
+
 	/*
 	 * A special case for gfs2 where we want to allow lvextend
 	 * of an LV that has an existing shared lock, which is normally

lib/locking/lvmlockd.h

@@ -103,6 +103,8 @@ int lockd_lv_uses_lock(struct logical_volume *lv);
 
 int lockd_lv_refresh(struct cmd_context *cmd, struct lvresize_params *lp);
 
+int lockd_query_lv(struct cmd_context *cmd, struct logical_volume *lv, int *ex, int *sh);
+
 #else /* LVMLOCKD_SUPPORT */
 
 static inline void lvmlockd_set_socket(const char *sock)
@@ -258,6 +260,11 @@ static inline int lockd_lv_refresh(struct cmd_context *cmd, struct lvresize_para
 	return 0;
 }
 
+static inline int lockd_query_lv(struct cmd_context *cmd, struct logical_volume *lv, int *ex, int *sh)
+{
+	return 0;
+}
+
 #endif	/* LVMLOCKD_SUPPORT */
 
 #endif	/* _LVMLOCKD_H */