report: cachevol field for writecache LVs

Use dm-writecache to attach a cache volume (CV) to an LV.
The LV type becomes "writecache" (--type writecache can
be optionally specified when attaching.)

  lvconvert --cachevol-attach CV LV
  lvconvert --cachevol-detach CV LV

The cachevol (CV) is a special LV that was created on
cache devices, e.g.

  lvcreate --cachevol -L <size> -n <name> VG

Cache devices are special PVs that are added to the
VG to use for cachevols, e.g.

  vgextend --cachedev VG PV

Example:

  Add persistent memory to a VG as a cache device:
  vgextend --cachedev vg /dev/pmem0

  Create a cache volume from pmem0:
  lvcreate --cachevol -L1G -n cvol0 vg

  Create a second cache volume from pmem0:
  lvcreate --cachevol -L1G -n cvol1 vg

  Attach the cache volumes to LVs:
  lvconvert --cachevol-attach cvol0 vg/lv0
  lvconvert --cachevol-attach cvol1 vg/lv1

device_mapper/all.h

@@ -378,6 +378,16 @@ struct dm_status_cache {
 int dm_get_status_cache(struct dm_pool *mem, const char *params,
 			struct dm_status_cache **status);
 
+struct dm_status_writecache {
+	uint32_t error;
+	uint64_t total_blocks;
+	uint64_t free_blocks;
+	uint64_t writeback_blocks;
+};
+
+int dm_get_status_writecache(struct dm_pool *mem, const char *params,
+                             struct dm_status_writecache **status);
+
 /*
  * Parse params from STATUS call for snapshot target
  *
@@ -914,6 +924,13 @@ int dm_tree_node_add_cache_target(struct dm_tree_node *node,
 				  const struct dm_config_node *policy_settings,
 				  uint32_t data_block_size);
 
+int dm_tree_node_add_writecache_target(struct dm_tree_node *node,
+				uint64_t size,
+				const char *origin_uuid,
+				const char *cache_uuid,
+				int pmem);
+
+
 /*
  * VDO target
  */

device_mapper/libdm-deptree.c

@@ -38,6 +38,7 @@ enum {
 	SEG_SNAPSHOT_MERGE,
 	SEG_STRIPED,
 	SEG_ZERO,
+	SEG_WRITECACHE,
 	SEG_THIN_POOL,
 	SEG_THIN,
 	SEG_VDO,
@@ -77,6 +78,7 @@ static const struct {
 	{ SEG_SNAPSHOT_MERGE, "snapshot-merge" },
 	{ SEG_STRIPED, "striped" },
 	{ SEG_ZERO, "zero"},
+	{ SEG_WRITECACHE, "writecache"},
 	{ SEG_THIN_POOL, "thin-pool"},
 	{ SEG_THIN, "thin"},
 	{ SEG_VDO, "vdo" },
@@ -208,6 +210,9 @@ struct load_segment {
 	struct dm_tree_node *vdo_data;  /* VDO */
 	struct dm_vdo_target_params vdo_params; /* VDO */
 	const char *vdo_name;           /* VDO - device name is ALSO passed as table arg */
+
+	struct dm_tree_node *cachevol;	/* writecache */
+	int cachevol_pmem;
 };
 
 /* Per-device properties */
@@ -2597,6 +2602,27 @@ static int _cache_emit_segment_line(struct dm_task *dmt,
 	return 1;
 }
 
+static int _writecache_emit_segment_line(struct dm_task *dmt,
+				    struct load_segment *seg,
+				    char *params, size_t paramsize)
+{
+	int pos = 0;
+	char origin_dev[DM_FORMAT_DEV_BUFSIZE];
+	char cache_dev[DM_FORMAT_DEV_BUFSIZE];
+
+	if (!_build_dev_string(origin_dev, sizeof(origin_dev), seg->origin))
+		return_0;
+
+	if (!_build_dev_string(cache_dev, sizeof(cache_dev), seg->cachevol))
+		return_0;
+
+	EMIT_PARAMS(pos, "%s %s %s 4096 0",
+		    seg->cachevol_pmem ? "p" : "s",
+		    origin_dev, cache_dev);
+
+	return 1;
+}
+
 static int _thin_pool_emit_segment_line(struct dm_task *dmt,
 					struct load_segment *seg,
 					char *params, size_t paramsize)
@@ -2776,6 +2802,10 @@ static int _emit_segment_line(struct dm_task *dmt, uint32_t major,
 		if (!_cache_emit_segment_line(dmt, seg, params, paramsize))
 			return_0;
 		break;
+	case SEG_WRITECACHE:
+		if (!_writecache_emit_segment_line(dmt, seg, params, paramsize))
+			return_0;
+		break;
 	}
 
 	switch(seg->type) {
@@ -2787,6 +2817,7 @@ static int _emit_segment_line(struct dm_task *dmt, uint32_t major,
 	case SEG_THIN_POOL:
 	case SEG_THIN:
 	case SEG_CACHE:
+	case SEG_WRITECACHE:
 		break;
 	case SEG_CRYPT:
 	case SEG_LINEAR:
@@ -3567,6 +3598,38 @@ int dm_tree_node_add_cache_target(struct dm_tree_node *node,
 	return 1;
 }
 
+int dm_tree_node_add_writecache_target(struct dm_tree_node *node,
+				  uint64_t size,
+				  const char *origin_uuid,
+				  const char *cache_uuid,
+				  int pmem)
+{
+	struct load_segment *seg;
+
+	if (!(seg = _add_segment(node, SEG_WRITECACHE, size)))
+		return_0;
+
+	seg->cachevol_pmem = pmem;
+
+	if (!(seg->cachevol = dm_tree_find_node_by_uuid(node->dtree,
+							cache_uuid))) {
+		log_error("Missing writecache's cachevol uuid %s.", cache_uuid);
+		return 0;
+	}
+	if (!_link_tree_nodes(node, seg->cachevol))
+		return_0;
+
+	if (!(seg->origin = dm_tree_find_node_by_uuid(node->dtree,
+						      origin_uuid))) {
+		log_error("Missing writecache's origin uuid %s.", origin_uuid);
+		return 0;
+	}
+	if (!_link_tree_nodes(node, seg->origin))
+		return_0;
+
+	return 1;
+}
+
 int dm_tree_node_add_replicator_target(struct dm_tree_node *node,
 				       uint64_t size,
 				       const char *rlog_uuid,

device_mapper/libdm-targets.c

@@ -346,6 +346,38 @@ bad:
 	return 0;
 }
 
+/*
+ * From linux/Documentation/device-mapper/writecache.txt
+ *
+ * Status:
+ * 1. error indicator - 0 if there was no error, otherwise error number
+ * 2. the number of blocks
+ * 3. the number of free blocks
+ * 4. the number of blocks under writeback
+ */
+
+int dm_get_status_writecache(struct dm_pool *mem, const char *params,
+			     struct dm_status_writecache **status)
+{
+	struct dm_status_writecache *s;
+
+	if (!(s = dm_pool_zalloc(mem, sizeof(struct dm_status_writecache))))
+		return_0;
+
+	if (sscanf(params, "%u %llu %llu %llu",
+		   &s->error,
+		   (unsigned long long *)&s->total_blocks,
+		   (unsigned long long *)&s->free_blocks,
+		   (unsigned long long *)&s->writeback_blocks) != 4) {
+		log_error("Failed to parse writecache params: %s.", params);
+		dm_pool_free(mem, s);
+		return 0;
+	}
+
+	*status = s;
+	return 1;
+}
+
 int parse_thin_pool_status(const char *params, struct dm_status_thin_pool *s)
 {
 	int pos;

lib/Makefile.in

@@ -19,6 +19,7 @@ top_builddir = @top_builddir@
 SOURCES =\
 	activate/activate.c \
 	cache/lvmcache.c \
+	writecache/writecache.c \
 	cache_segtype/cache.c \
 	commands/toolcontext.c \
 	config/config.c \

lib/activate/activate.h

@@ -38,6 +38,7 @@ typedef enum {
 	SEG_STATUS_THIN,
 	SEG_STATUS_THIN_POOL,
 	SEG_STATUS_VDO_POOL,
+	SEG_STATUS_WRITECACHE,
 	SEG_STATUS_UNKNOWN
 } lv_seg_status_type_t;
 
@@ -51,6 +52,7 @@ struct lv_seg_status {
 		struct dm_status_snapshot *snapshot;
 		struct dm_status_thin *thin;
 		struct dm_status_thin_pool *thin_pool;
+		struct dm_status_writecache *writecache;
 		struct lv_status_vdo vdo_pool;
 	};
 };
@@ -254,6 +256,7 @@ int device_is_usable(struct device *dev, struct dev_usable_check_params check);
 void fs_unlock(void);
 
 #define TARGET_NAME_CACHE "cache"
+#define TARGET_NAME_WRITECACHE "writecache"
 #define TARGET_NAME_ERROR "error"
 #define TARGET_NAME_ERROR_OLD "erro"	/* Truncated in older kernels */
 #define TARGET_NAME_LINEAR "linear"
@@ -270,6 +273,7 @@ void fs_unlock(void);
 
 #define MODULE_NAME_CLUSTERED_MIRROR "clog"
 #define MODULE_NAME_CACHE TARGET_NAME_CACHE
+#define MODULE_NAME_WRITECACHE TARGET_NAME_WRITECACHE
 #define MODULE_NAME_ERROR TARGET_NAME_ERROR
 #define MODULE_NAME_LOG_CLUSTERED "log-clustered"
 #define MODULE_NAME_LOG_USERSPACE "log-userspace"

lib/activate/dev_manager.c

@@ -213,6 +213,10 @@ static int _get_segment_status_from_target_params(const char *target_name,
 		if (!parse_vdo_pool_status(seg_status->mem, seg->lv, params, &seg_status->vdo_pool))
 			return_0;
 		seg_status->type = SEG_STATUS_VDO_POOL;
+	} else if (segtype_is_writecache(segtype)) {
+		if (!dm_get_status_writecache(seg_status->mem, params, &(seg_status->writecache)))
+			return_0;
+		seg_status->type = SEG_STATUS_WRITECACHE;
 	} else
 		/*
 		 * TODO: Add support for other segment types too!
@@ -2880,6 +2884,10 @@ static int _add_segment_to_dtree(struct dev_manager *dm,
 	    !_add_new_lv_to_dtree(dm, dtree, seg->metadata_lv, laopts, NULL))
 		return_0;
 
+	if (seg->cachevol &&
+	    !_add_new_lv_to_dtree(dm, dtree, seg->cachevol, laopts, NULL))
+			return_0;
+
 	/* Add pool layer */
 	if (seg->pool_lv && !laopts->origin_only &&
 	    !_add_new_lv_to_dtree(dm, dtree, seg->pool_lv, laopts,

lib/cache/lvmcache.c

@@ -64,6 +64,7 @@ static struct dm_hash_table *_pvid_hash = NULL;
 static struct dm_hash_table *_vgid_hash = NULL;
 static struct dm_hash_table *_vgname_hash = NULL;
 static DM_LIST_INIT(_vginfos);
+static DM_LIST_INIT(_cachedevs);
 static DM_LIST_INIT(_found_duplicate_devs);
 static DM_LIST_INIT(_unused_duplicate_devs);
 static int _scanning_in_progress = 0;
@@ -1075,11 +1076,15 @@ static struct device *_device_from_pvid(const struct id *pvid, uint64_t *label_s
 struct device *lvmcache_device_from_pvid(struct cmd_context *cmd, const struct id *pvid, uint64_t *label_sector)
 {
 	struct device *dev;
+	struct cachedev *cd;
 
 	dev = _device_from_pvid(pvid, label_sector);
 	if (dev)
 		return dev;
 
+	if ((cd = lvmcache_cachedev_from_pvid(pvid)))
+		return cd->dev;
+
 	log_debug_devs("No device with uuid %s.", (const char *)pvid);
 	return NULL;
 }
@@ -1563,6 +1568,41 @@ int lvmcache_update_vgname_and_id(struct lvmcache_info *info, struct lvmcache_vg
 	return 1;
 }
 
+/*
+ * The CDs have no metadata, so the scan put the info structs for
+ * them into the orphan vg.  Using the VG metadata, those CD info
+ * structs can now be identified and removed, and cachedev entries
+ * created.
+ */
+void lvmcache_update_vg_cachedevs(struct volume_group *vg)
+{
+	struct pv_list *pvl;
+	struct lvmcache_info *info;
+	struct cachedev *cd;
+
+	dm_list_iterate_items(pvl, &vg->cds) {
+		if (!(info = lvmcache_info_from_pvid((const char *)&pvl->pv->id, pvl->pv->dev, 0))) {
+			log_debug("lvmcache missing info for cachedev pv %s", dev_name(pvl->pv->dev));
+			continue;
+		}
+
+		if (!lvmcache_cachedev_from_pvid(&pvl->pv->id)) {
+			if (!(cd = zalloc(sizeof(*cd)))) {
+				stack;
+				return;
+			}
+
+			log_debug("lvmcache add cachedev %s %s", dev_name(pvl->pv->dev), pvl->pv->dev->pvid);
+			cd->dev = pvl->pv->dev;
+			cd->device_size = info->device_size;
+			cd->vg_name = strdup(vg->name);
+			dm_list_add(&_cachedevs, &cd->list);
+		}
+
+		lvmcache_del(info);
+	}
+}
+
 int lvmcache_update_vg(struct volume_group *vg, unsigned precommitted)
 {
 	struct pv_list *pvl;
@@ -2280,3 +2320,22 @@ int lvmcache_scan_mismatch(struct cmd_context *cmd, const char *vgname, const ch
 	return 1;
 }
 
+struct cachedev *lvmcache_cachedev_from_pvid(const struct id *pvid)
+{
+	struct cachedev *cd;
+
+	dm_list_iterate_items(cd, &_cachedevs) {
+		if (!memcmp(&cd->dev->pvid, pvid, sizeof(struct id))) {
+			log_debug("lvmcache_cachedev_from_pvid %s %s", dev_name(cd->dev), cd->dev->pvid);
+			return cd;
+		}
+	}
+	return NULL;
+}
+
+void lvmcache_del_cachedev(struct cachedev *cd)
+{
+	log_debug("lvmcache del cachedev %s", dev_name(cd->dev));
+	dm_list_del(&cd->list);
+}
+

lib/cache/lvmcache.h

@@ -209,4 +209,8 @@ void lvmcache_drop_saved_vgid(const char *vgid);
 
 int dev_in_device_list(struct device *dev, struct dm_list *head);
 
+struct cachedev *lvmcache_cachedev_from_pvid(const struct id *pvid);
+void lvmcache_update_vg_cachedevs(struct volume_group *vg);
+void lvmcache_del_cachedev(struct cachedev *cd);
+
 #endif

lib/commands/toolcontext.c

@@ -1366,6 +1366,8 @@ static int _init_segtypes(struct cmd_context *cmd)
 	if (!init_vdo_segtypes(cmd, &seglib))
 		return_0;
 #endif
+	if (!init_writecache_segtypes(cmd, &seglib))
+		return 0;
 
 #ifdef HAVE_LIBDL
 	/* Load any formats in shared libs unless static */

lib/config/defaults.h

@@ -264,7 +264,7 @@
 #define DEFAULT_DEVTYPES_COLS "devtype_name,devtype_max_partitions,devtype_description"
 #define DEFAULT_COMMAND_LOG_COLS "log_seq_num,log_type,log_context,log_object_type,log_object_name,log_object_id,log_object_group,log_object_group_id,log_message,log_errno,log_ret_code"
 
-#define DEFAULT_LVS_COLS_VERB "lv_name,vg_name,seg_count,lv_attr,lv_size,lv_major,lv_minor,lv_kernel_major,lv_kernel_minor,pool_lv,origin,data_percent,metadata_percent,move_pv,copy_percent,mirror_log,convert_lv,lv_uuid,lv_profile"
+#define DEFAULT_LVS_COLS_VERB "lv_name,vg_name,seg_count,lv_attr,lv_size,lv_major,lv_minor,lv_kernel_major,lv_kernel_minor,cachevol,pool_lv,origin,data_percent,metadata_percent,move_pv,copy_percent,mirror_log,convert_lv,lv_uuid,lv_profile"
 #define DEFAULT_VGS_COLS_VERB "vg_name,vg_attr,vg_extent_size,pv_count,lv_count,snap_count,vg_size,vg_free,vg_uuid,vg_profile"
 #define DEFAULT_PVS_COLS_VERB "pv_name,vg_name,pv_fmt,pv_attr,pv_size,pv_free,dev_size,pv_uuid"
 #define DEFAULT_SEGS_COLS_VERB "lv_name,vg_name,lv_attr,seg_start,seg_size,stripes,segtype,stripesize,chunksize"

lib/device/device.h

@@ -70,6 +70,7 @@ struct device {
 	int read_ahead;
 	int bcache_fd;
 	uint32_t flags;
+	uint32_t is_pmem:1;
 	unsigned size_seqno;
 	uint64_t size;
 	uint64_t end;
@@ -110,6 +111,13 @@ struct device_area {
 	uint64_t size;		/* Bytes */
 };
 
+struct cachedev {
+	struct dm_list list;
+	struct device *dev;
+	uint64_t device_size;   /* Bytes */
+	const char *vg_name;
+};
+
 /*
  * Support for external device info.
  */

lib/format_text/export.c

@@ -520,59 +520,69 @@ static const char *_get_pv_name(struct formatter *f, struct physical_volume *pv)
 	return _get_pv_name_from_uuid(f, uuid);
 }
 
+static int _print_pv(struct formatter *f, struct volume_group *vg, struct physical_volume *pv)
+{
+	char buffer[PATH_MAX * 2];
+	const char *name;
+
+	if (!id_write_format(&pv->id, buffer, sizeof(buffer)))
+		return_0;
+
+	if (!(name = _get_pv_name_from_uuid(f, buffer)))
+		return_0;
+
+	outnl(f);
+	outf(f, "%s {", name);
+	_inc_indent(f);
+
+	outf(f, "id = \"%s\"", buffer);
+
+	if (strlen(pv_dev_name(pv)) >= PATH_MAX) {
+		log_error("pv device name size is out of bounds.");
+		return 0;
+	}
+
+	outhint(f, "device = \"%s\"",
+		dm_escape_double_quotes(buffer, pv_dev_name(pv)));
+	outnl(f);
+
+	if (!_print_flag_config(f, pv->status, PV_FLAGS))
+		return_0;
+
+	if (!_out_list(f, &pv->tags, "tags"))
+		return_0;
+
+	outsize(f, pv->size, "dev_size = " FMTu64, pv->size);
+
+	outf(f, "pe_start = " FMTu64, pv->pe_start);
+	outsize(f, vg->extent_size * (uint64_t) pv->pe_count,
+		"pe_count = %u", pv->pe_count);
+
+	if (pv->ba_start && pv->ba_size) {
+		outf(f, "ba_start = " FMTu64, pv->ba_start);
+		outsize(f, pv->ba_size, "ba_size = " FMTu64, pv->ba_size);
+	}
+
+	_dec_indent(f);
+	outf(f, "}");
+	return 1;
+}
+
 static int _print_pvs(struct formatter *f, struct volume_group *vg)
 {
 	struct pv_list *pvl;
-	struct physical_volume *pv;
-	char buffer[PATH_MAX * 2];
-	const char *name;
 
 	outf(f, "physical_volumes {");
 	_inc_indent(f);
 
 	dm_list_iterate_items(pvl, &vg->pvs) {
-		pv = pvl->pv;
-
-		if (!id_write_format(&pv->id, buffer, sizeof(buffer)))
+		if (!_print_pv(f, vg, pvl->pv))
 			return_0;
+	}
 
-		if (!(name = _get_pv_name_from_uuid(f, buffer)))
+	dm_list_iterate_items(pvl, &vg->cds) {
+		if (!_print_pv(f, vg, pvl->pv))
 			return_0;
-
-		outnl(f);
-		outf(f, "%s {", name);
-		_inc_indent(f);
-
-		outf(f, "id = \"%s\"", buffer);
-
-		if (strlen(pv_dev_name(pv)) >= PATH_MAX) {
-			log_error("pv device name size is out of bounds.");
-			return 0;
-		}
-
-		outhint(f, "device = \"%s\"",
-			dm_escape_double_quotes(buffer, pv_dev_name(pv)));
-		outnl(f);
-
-		if (!_print_flag_config(f, pv->status, PV_FLAGS))
-			return_0;
-
-		if (!_out_list(f, &pv->tags, "tags"))
-			return_0;
-
-		outsize(f, pv->size, "dev_size = " FMTu64, pv->size);
-
-		outf(f, "pe_start = " FMTu64, pv->pe_start);
-		outsize(f, vg->extent_size * (uint64_t) pv->pe_count,
-			"pe_count = %u", pv->pe_count);
-
-		if (pv->ba_start && pv->ba_size) {
-			outf(f, "ba_start = " FMTu64, pv->ba_start);
-			outsize(f, pv->ba_size, "ba_size = " FMTu64, pv->ba_size);
-		}
-
-		_dec_indent(f);
-		outf(f, "}");
 	}
 
 	_dec_indent(f);
@@ -971,6 +981,23 @@ static int _build_pv_names(struct formatter *f, struct volume_group *vg)
 			return_0;
 	}
 
+	dm_list_iterate_items(pvl, &vg->cds) {
+		pv = pvl->pv;
+
+		if (dm_snprintf(buffer, sizeof(buffer), "pv%d", count++) < 0)
+			return_0;
+
+		if (!(name = dm_pool_strdup(f->mem, buffer)))
+			return_0;
+
+		if (!(uuid = dm_pool_zalloc(f->mem, 64)) ||
+		   !id_write_format(&pv->id, uuid, 64))
+			return_0;
+
+		if (!dm_hash_insert(f->pv_names, uuid, name))
+			return_0;
+	}
+
 	return 1;
 }
 

lib/format_text/flags.c

@@ -48,6 +48,7 @@ static const struct flag _pv_flags[] = {
 	{EXPORTED_VG, "EXPORTED", STATUS_FLAG},
 	{MISSING_PV, "MISSING", COMPATIBLE_FLAG},
 	{MISSING_PV, "MISSING", STATUS_FLAG},
+	{CACHEDEV_PV, "CACHEDEV", STATUS_FLAG},
 	{PV_MOVED_VG, NULL, 0},
 	{UNLABELLED_PV, NULL, 0},
 	{0, NULL, 0}
@@ -61,6 +62,7 @@ static const struct flag _lv_flags[] = {
 	{VISIBLE_LV, "VISIBLE", STATUS_FLAG},
 	{PVMOVE, "PVMOVE", STATUS_FLAG},
 	{LOCKED, "LOCKED", STATUS_FLAG},
+	{CACHEVOL, "CACHEVOL", STATUS_FLAG},
 	{LV_NOTSYNCED, "NOTSYNCED", STATUS_FLAG},
 	{LV_REBUILD, "REBUILD", STATUS_FLAG},
 	{LV_RESHAPE, "RESHAPE", SEGTYPE_FLAG},
@@ -101,6 +103,7 @@ static const struct flag _lv_flags[] = {
 	{LV_VDO, NULL, 0},
 	{LV_VDO_POOL, NULL, 0},
 	{LV_VDO_POOL_DATA, NULL, 0},
+	{WRITECACHE, NULL, 0},
 	{LV_PENDING_DELETE, NULL, 0}, /* FIXME Display like COMPATIBLE_FLAG */
 	{LV_REMOVED, NULL, 0},
 	{0, NULL, 0}

lib/format_text/format-text.c

@@ -597,6 +597,8 @@ static int _vg_write_raw(struct format_instance *fid, struct volume_group *vg,
 	uint64_t new_wrap = 0, old_wrap = 0, new_end;
 	int found = 0;
 	int noprecommit = 0;
+	uint32_t meta_size;
+	char *meta_buf;
 	const char *old_vg_name = NULL;
 
 	/* Ignore any mda on a PV outside the VG. vgsplit relies on this */
@@ -615,11 +617,20 @@ static int _vg_write_raw(struct format_instance *fid, struct volume_group *vg,
 	if (!(mdah = raw_read_mda_header(fid->fmt, &mdac->area, mda_is_primary(mda))))
 		goto_out;
 
-	if (!fidtc->raw_metadata_buf &&
-	    !(fidtc->raw_metadata_buf_size =
-			text_vg_export_raw(vg, "", &fidtc->raw_metadata_buf))) {
-		log_error("VG %s metadata writing failed", vg->name);
-		goto out;
+	/*
+	 * The first time through, for the first mda, we create a new export
+	 * buffer, and subsequent mdas use the same.
+	 */
+	if (!fidtc->raw_metadata_buf) {
+		meta_size = text_vg_export_raw(vg, "", &meta_buf);
+
+		if (!meta_size || !meta_buf) {
+			log_error("VG %s metadata writing failed", vg->name);
+			goto out;
+		}
+
+		fidtc->raw_metadata_buf = meta_buf;
+		fidtc->raw_metadata_buf_size = meta_size;
 	}
 
 	rlocn = _read_metadata_location_vg(&mdac->area, mdah, mda_is_primary(mda), old_vg_name ? : vg->name, &noprecommit);

lib/format_text/import_vsn1.c

@@ -231,6 +231,12 @@ static int _read_pv(struct format_instance *fid,
 		return 0;
 	}
 
+	if (pv->status & CACHEDEV_PV) {
+		pv->is_cachedev = 1;
+		if (pv->dev)
+			set_cachedev_type(pv);
+	}
+
 	if (!pv->dev)
 		pv->status |= MISSING_PV;
 
@@ -310,9 +316,13 @@ static int _read_pv(struct format_instance *fid,
 	if (!alloc_pv_segment_whole_pv(mem, pv))
 		return_0;
 
-	vg->extent_count += pv->pe_count;
-	vg->free_count += pv->pe_count;
-	add_pvl_to_vgs(vg, pvl);
+	if (!pv->is_cachedev) {
+		vg->extent_count += pv->pe_count;
+		vg->free_count += pv->pe_count;
+		add_pvl_to_vgs(vg, pvl);
+	} else {
+		dm_list_add(&vg->cds, &pvl->list);
+	}
 
 	return 1;
 }
@@ -984,6 +994,7 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 	const char *str, *format_str, *system_id;
 	struct volume_group *vg;
 	struct dm_hash_table *pv_hash = NULL, *lv_hash = NULL;
+	struct lv_list *lvl;
 	uint64_t vgstatus;
 
 	/* skip any top-level values */
@@ -1178,6 +1189,11 @@ static struct volume_group *_read_vg(struct format_instance *fid,
 		goto bad;
 	}
 
+	dm_list_iterate_items(lvl, &vg->lvs) {
+		if (lv_is_cachevol(lvl->lv))
+			set_cachevol_type(lvl->lv);
+	}
+
 	dm_hash_destroy(pv_hash);
 	dm_hash_destroy(lv_hash);
 

lib/label/label.c

@@ -194,6 +194,13 @@ int label_write(struct device *dev, struct label *label)
 	lh->sector_xl = xlate64(label->sector);
 	lh->offset_xl = xlate32(sizeof(*lh));
 
+	/*
+	 * Set pv_header and pv_header_extension fields before
+	 * calculating the crc of the block to set in lh crc.
+	 * The crc covers data from just after the crc field
+	 * to the end of the sector.
+	 */
+
 	if (!(label->labeller->ops->write)(label, buf))
 		return_0;
 

lib/metadata/lv.c

@@ -576,6 +576,8 @@ struct logical_volume *lv_origin_lv(const struct logical_volume *lv)
 		origin = first_seg(lv)->origin;
 	else if (lv_is_thin_volume(lv) && first_seg(lv)->external_lv)
 		origin = first_seg(lv)->external_lv;
+	else if (lv_is_writecache(lv) && first_seg(lv)->origin)
+		origin = first_seg(lv)->origin;
 
 	return origin;
 }
@@ -699,6 +701,37 @@ char *lv_mirror_log_uuid_dup(struct dm_pool *mem, const struct logical_volume *l
 	return _do_lv_mirror_log_dup(mem, lv, 1);
 }
 
+struct logical_volume *lv_cachevol_lv(const struct logical_volume *lv)
+{
+	if (lv_is_writecache(lv))
+		return first_seg(lv)->cachevol;
+	return NULL;
+}
+
+static char *_do_lv_cachevol_dup(struct dm_pool *mem, const struct logical_volume *lv,
+				    int uuid)
+{
+	struct logical_volume *cachevol_lv = lv_cachevol_lv(lv);
+
+	if (!cachevol_lv)
+		return NULL;
+
+	if (uuid)
+		return lv_uuid_dup(mem, cachevol_lv);
+
+	return lv_name_dup(mem, cachevol_lv);
+}
+
+char *lv_cachevol_dup(struct dm_pool *mem, const struct logical_volume *lv)
+{
+	return _do_lv_cachevol_dup(mem, lv, 0);
+}
+
+char *lv_cachevol_uuid_dup(struct dm_pool *mem, const struct logical_volume *lv)
+{
+	return _do_lv_cachevol_dup(mem, lv, 1);
+}
+
 struct logical_volume *lv_pool_lv(const struct logical_volume *lv)
 {
 	if (lv_is_thin_volume(lv) || lv_is_cache(lv))
@@ -1182,7 +1215,7 @@ char *lv_attr_dup_with_info_and_seg_status(struct dm_pool *mem, const struct lv_
 		 lv_is_pool_metadata_spare(lv) ||
 		 lv_is_raid_metadata(lv))
 		repstr[0] = 'e';
-	else if (lv_is_cache_type(lv))
+	else if (lv_is_cache_type(lv) || lv_is_cachevol(lv) || lv_is_writecache(lv))
 		repstr[0] = 'C';
 	else if (lv_is_raid(lv))
 		repstr[0] = (lv_is_not_synced(lv)) ? 'R' : 'r';

lib/metadata/lv.h

@@ -57,6 +57,7 @@ struct logical_volume {
 
 	uint64_t timestamp;
 	unsigned new_lock_args:1;
+	unsigned cachevol_pmem:1;
 	const char *hostname;
 	const char *lock_args;
 };
@@ -116,6 +117,7 @@ struct logical_volume *lv_mirror_log(const struct logical_volume *lv);
 struct logical_volume *lv_data(const struct logical_volume *lv);
 struct logical_volume *lv_metadata_lv(const struct logical_volume *lv);
 struct logical_volume *lv_pool_lv(const struct logical_volume *lv);
+struct logical_volume *lv_cachevol_lv(const struct logical_volume *lv);
 
 /* LV properties */
 uint64_t lv_size(const struct logical_volume *lv);
@@ -193,6 +195,8 @@ char *lv_profile_dup(struct dm_pool *mem, const struct logical_volume *lv);
 char *lv_lock_args_dup(struct dm_pool *mem, const struct logical_volume *lv);
 char *lvseg_kernel_discards_dup_with_info_and_seg_status(struct dm_pool *mem, const struct lv_with_info_and_seg_status *lvdm);
 char *lv_time_dup(struct dm_pool *mem, const struct logical_volume *lv, int iso_mode);
+char *lv_cachevol_dup(struct dm_pool *mem, const struct logical_volume *lv);
+char *lv_cachevol_uuid_dup(struct dm_pool *mem, const struct logical_volume *lv);
 
 typedef enum {
 	PERCENT_GET_DATA = 0,

lib/metadata/lv_manip.c

@@ -1387,6 +1387,11 @@ static int _lv_reduce(struct logical_volume *lv, uint32_t extents, int delete)
 		data_copies = seg->data_copies;
 	}
 
+	if (lv_is_cachevol(lv) && !dm_list_empty(&lv->segs_using_this_lv)) {
+		log_error("Cachevol LV must be detached before being removed.");
+		return 0;
+	}
+
 	if (lv_is_merging_origin(lv)) {
 		log_debug_metadata("Dropping snapshot merge of %s to removed origin %s.",
 				   find_snapshot(lv)->lv->name, lv->name);
@@ -1572,6 +1577,11 @@ int lv_reduce(struct logical_volume *lv, uint32_t extents)
 {
 	struct lv_segment *seg = first_seg(lv);
 
+	if (lv_is_writecache(lv)) {
+		log_error("Remove not yet allowed on LVs with an attached cachevol LV.");
+		return 0;
+	}
+
 	/* Ensure stripe boundary extents on RAID LVs */
 	if (lv_is_raid(lv) && extents != lv->le_count)
 		extents =_round_to_stripe_boundary(lv->vg, extents,
@@ -5561,6 +5571,15 @@ int lv_resize(struct logical_volume *lv,
 	int ret = 0;
 	int status;
 
+	if (lv_is_writecache(lv)) {
+		log_error("Resize not yet allowed on LVs with an attached cachevol LV.");
+		return 0;
+	}
+	if (lv_is_cachevol(lv)) {
+		log_error("Resize not yet allowed on cachevol LVs.");
+		return 0;
+	}
+
 	if (!_lvresize_check(lv, lp))
 		return_0;
 
@@ -7760,6 +7779,9 @@ static struct logical_volume *_lv_create_an_lv(struct volume_group *vg,
 		log_debug_metadata("Setting read ahead sectors %u.", lv->read_ahead);
 	}
 
+	if (lp->cachevol)
+		lv->status |= CACHEVOL;
+
 	if (!segtype_is_pool(create_segtype) &&
 	    !segtype_is_vdo_pool(create_segtype) &&
 	    lp->minor >= 0) {

lib/metadata/merge.c

@@ -696,6 +696,8 @@ int check_lv_segments(struct logical_volume *lv, int complete_vg)
 			seg_found++;
 		if (seg_is_thin_volume(seg) && (seg->origin == lv || seg->external_lv == lv))
 			seg_found++;
+		if (seg->cachevol == lv)
+			seg_found++;
 
 		if (!seg_found) {
 			log_error("LV %s is used by LV %s:%" PRIu32 "-%" PRIu32

lib/metadata/metadata-exported.h

@@ -56,7 +56,7 @@
 #define ALLOCATABLE_PV		UINT64_C(0x0000000000000008)	/* PV */
 #define ARCHIVED_VG		ALLOCATABLE_PV		/* VG, reuse same bit */
 
-//#define SPINDOWN_LV		UINT64_C(0x0000000000000010)	/* LV */
+#define CACHEDEV_PV		UINT64_C(0x0000000000000010)	/* PV */
 //#define BADBLOCK_ON		UINT64_C(0x0000000000000020)	/* LV */
 #define VISIBLE_LV		UINT64_C(0x0000000000000040)	/* LV */
 #define FIXED_MINOR		UINT64_C(0x0000000000000080)	/* LV */
@@ -151,6 +151,8 @@
 #define LV_VDO_POOL		UINT64_C(0x0000000040000000)    /* LV - Internal user only */
 #define LV_VDO_POOL_DATA	UINT64_C(0x8000000000000000)    /* LV - Internal user only */
 
+#define CACHEVOL		UINT64_C(0x0000000000000010)	/* LV - same bit as CACHEDEV pv flag */
+#define WRITECACHE		UINT64_C(0x0000000080000000)    /* LV - same bit as UNLABELLED_PV pv flag */
 
 /* Format features flags */
 #define FMT_SEGMENTS		0x00000001U	/* Arbitrary segment params? */
@@ -255,6 +257,7 @@
 #define lv_is_pool_metadata(lv)		(((lv)->status & (CACHE_POOL_METADATA | THIN_POOL_METADATA)) ? 1 : 0)
 #define lv_is_pool_metadata_spare(lv)	(((lv)->status & POOL_METADATA_SPARE) ? 1 : 0)
 #define lv_is_lockd_sanlock_lv(lv)	(((lv)->status & LOCKD_SANLOCK_LV) ? 1 : 0)
+#define lv_is_writecache(lv)	(((lv)->status & WRITECACHE) ? 1 : 0)
 
 #define lv_is_vdo(lv)		(((lv)->status & LV_VDO) ? 1 : 0)
 #define lv_is_vdo_pool(lv)	(((lv)->status & LV_VDO_POOL) ? 1 : 0)
@@ -263,6 +266,8 @@
 
 #define lv_is_removed(lv)	(((lv)->status & LV_REMOVED) ? 1 : 0)
 
+#define lv_is_cachevol(lv)	(((lv)->status & CACHEVOL) ? 1 : 0)
+
 /* Recognize component LV (matching lib/misc/lvm-string.c _lvname_has_reserved_component_string()) */
 #define lv_is_component(lv) (lv_is_cache_origin(lv) || ((lv)->status & (\
 	CACHE_POOL_DATA |\
@@ -502,6 +507,8 @@ struct lv_segment {
 	struct dm_vdo_target_params vdo_params;	/* For VDO-pool */
 	uint32_t vdo_pool_header_size;		/* For VDO-pool */
 	uint32_t vdo_pool_virtual_extents;	/* For VDO-pool */
+
+	struct logical_volume *cachevol;	/* For writecache */
 };
 
 #define seg_type(seg, s)	(seg)->areas[(s)].type
@@ -608,6 +615,7 @@ struct pvcreate_params {
 	unsigned is_remove : 1;         /* is removing PVs, not creating */
 	unsigned preserve_existing : 1;
 	unsigned check_failed : 1;
+	unsigned cachedev : 1;
 };
 
 struct lvresize_params {
@@ -748,6 +756,7 @@ int vg_remove(struct volume_group *vg);
 int vg_rename(struct cmd_context *cmd, struct volume_group *vg,
 	      const char *new_name);
 int vg_extend_each_pv(struct volume_group *vg, struct pvcreate_params *pp);
+int vg_extend_each_cd(struct volume_group *vg, struct pvcreate_params *pp);
 
 int vgreduce_single(struct cmd_context *cmd, struct volume_group *vg,
 			    struct physical_volume *pv, int commit);
@@ -922,6 +931,7 @@ struct lvcreate_params {
 	int thin_chunk_size_calc_policy;
 	unsigned suppress_zero_warn : 1;
 	unsigned needs_lockd_init : 1;
+	unsigned cachevol: 1;
 
 	const char *vg_name; /* only-used when VG is not yet opened (in /tools) */
 	const char *lv_name; /* all */
@@ -1018,6 +1028,8 @@ struct pv_list *find_pv_in_vg(const struct volume_group *vg,
 			      const char *pv_name);
 struct pv_list *find_pv_in_vg_by_uuid(const struct volume_group *vg,
 				      const struct id *id);
+struct pv_list *find_cd_in_vg(const struct volume_group *vg,
+                               const char *pv_name);
 
 /* Find an LV within a given VG */
 struct lv_list *find_lv_in_vg(const struct volume_group *vg,
@@ -1339,4 +1351,8 @@ int is_system_id_allowed(struct cmd_context *cmd, const char *system_id);
 
 int vg_strip_outdated_historical_lvs(struct volume_group *vg);
 
+int set_cachedev_type(struct physical_volume *pv);
+
+int set_cachevol_type(struct logical_volume *lv);
+
 #endif

lib/metadata/metadata.c

@@ -699,6 +699,76 @@ int vg_extend_each_pv(struct volume_group *vg, struct pvcreate_params *pp)
 	return 1;
 }
 
+int vg_extend_each_cd(struct volume_group *vg, struct pvcreate_params *pp)
+{
+	struct pv_list *pvl_pp;
+	struct pv_list *pvl_vg;
+	struct physical_volume *pv;
+	const char *pv_name;
+	uint64_t pe_count;
+	int used;
+
+	dm_list_iterate_items(pvl_pp, &pp->pvs) {
+		log_debug_metadata("Adding CD %s to VG %s.", pv_dev_name(pvl_pp->pv), vg->name);
+
+		pv = pvl_pp->pv;
+		pv_name = pv_dev_name(pv);
+
+		if ((used = is_used_pv(pv)) < 0)
+			continue;
+		if (used) {
+			log_error("PV %s is used by a VG", pv_name);
+			continue;
+		}
+		if (pv_uses_vg(pv, vg)) {
+			log_error("PV %s might be constructed from same VG.", pv_name);
+			continue;
+		}
+		if (find_pv_in_vg(vg, pv_name)) {
+			log_error("PV %s already found in VG.", pv_name);
+			continue;
+		}
+		if (find_pv_in_vg_by_uuid(vg, &pv->id)) {
+			log_error("PV %s UUID already found in VG.", pv_name);
+			continue;
+		}
+
+		if (!(pvl_vg = dm_pool_zalloc(vg->vgmem, sizeof(*pvl_vg))))
+			return_0;
+
+		if (!(pv->vg_name = dm_pool_strdup(vg->vgmem, vg->name)))
+			return_0;
+
+		memcpy(&pv->vgid, &vg->id, sizeof(vg->id));
+
+		if (!set_cachedev_type(pv))
+			return_0;
+
+		pv->is_cachedev = 1;
+		pv->status |= CACHEDEV_PV;
+		pv->status |= ALLOCATABLE_PV;
+
+		pv->pe_size = vg->extent_size;
+		pv->pe_alloc_count = 0;
+
+		pv->pe_start = 2048; /* FIXME? */
+
+		pe_count = (pv->size - pv->pe_start) / vg->extent_size;
+		if (pe_count > UINT32_MAX)
+			return_0;
+
+		pv->pe_count = (uint32_t)pe_count;
+
+		if (!alloc_pv_segment_whole_pv(vg->vgmem, pv))
+			return_0;
+
+		pvl_vg->pv = pv;
+		dm_list_add(&vg->cds, &pvl_vg->list);
+	}
+
+	return 1;
+}
+
 int lv_change_tag(struct logical_volume *lv, const char *tag, int add_tag)
 {
 	char *tag_new;
@@ -1519,6 +1589,22 @@ struct pv_list *find_pv_in_vg(const struct volume_group *vg,
 	return NULL;
 }
 
+struct pv_list *find_cd_in_vg(const struct volume_group *vg,
+			       const char *pv_name)
+{
+	struct pv_list *pvl;
+	struct device *dev = dev_cache_get(vg->cmd, pv_name, vg->cmd->filter);
+
+	if (!dev)
+		return NULL;
+
+	dm_list_iterate_items(pvl, &vg->cds)
+		if (pvl->pv->dev == dev)
+			return pvl;
+
+	return NULL;
+}
+
 struct pv_list *find_pv_in_pv_list(const struct dm_list *pl,
 				   const struct physical_volume *pv)
 {
@@ -2077,6 +2163,10 @@ static int _lv_validate_references_single(struct logical_volume *lv, void *data)
 	unsigned s;
 	int r = 1;
 
+	/* FIXME: check vg->cds */
+	if (lv_is_cachevol(lv))
+		return 1;
+
 	if (lv != dm_hash_lookup_binary(vhash->lvid, &lv->lvid.id[1],
 					sizeof(lv->lvid.id[1]))) {
 		log_error(INTERNAL_ERROR
@@ -3775,6 +3865,7 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 		     !(vg = mda->ops->vg_read_precommit(fid, vgname, mda, &vg_fmtdata, &use_previous_vg)) && !use_previous_vg) ||
 		    (!use_precommitted &&
 		     !(vg = mda->ops->vg_read(fid, vgname, mda, &vg_fmtdata, &use_previous_vg)) && !use_previous_vg)) {
+			log_debug("inconsistent vg_read %s from text read", vgname);
 			inconsistent = 1;
 			vg_fmtdata = NULL;
 			continue;
@@ -3814,6 +3905,11 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 	}
 	fid->ref_count--;
 
+	if (correct_vg && !dm_list_empty(&correct_vg->cds)) {
+		log_debug("vg_read %s updating cds in lvmcache", vgname);
+		lvmcache_update_vg_cachedevs(correct_vg);
+	}
+
 	/* Ensure every PV in the VG was in the cache */
 	if (correct_vg) {
 		/*
@@ -3829,6 +3925,10 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 		 */
 		if (!inconsistent &&
 		    dm_list_size(&correct_vg->pvs) > dm_list_size(pvids)) {
+
+			log_debug("vg_read %s mismatch pvs %d infos %d",
+				  vgname, dm_list_size(&correct_vg->pvs), dm_list_size(pvids));
+
 			dm_list_iterate_items(pvl, &correct_vg->pvs) {
 				if (!pvl->pv->dev) {
 					inconsistent_pvs = 1;
@@ -3845,6 +3945,8 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 				 */
 				if (!(info = lvmcache_info_from_pvid(pvl->pv->dev->pvid, pvl->pv->dev, 1)) ||
 				    !lvmcache_is_orphan(info)) {
+					log_debug("inconsistent vg_read %s bad lvmcache info %p for %s",
+						  vgname, info, dev_name(pvl->pv->dev));
 					inconsistent_pvs = 1;
 					break;
 				}
@@ -4055,6 +4157,8 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 	lvmcache_update_vg(correct_vg, (correct_vg->status & PRECOMMITTED));
 
 	if (inconsistent) {
+		log_debug("inconsistent vg_read %s", vgname);
+
 		/* FIXME Test should be if we're *using* precommitted metadata not if we were searching for it */
 		if (use_precommitted) {
 			log_error("Inconsistent pre-commit metadata copies "
@@ -4163,8 +4267,10 @@ static struct volume_group *_vg_read(struct cmd_context *cmd,
 		}
 	}
 
-	if (inconsistent_pvs)
+	if (inconsistent_pvs) {
+		log_debug("inconsistent pvs in vg_read %s", vgname);
 		*mdas_consistent = 0;
+	}
 
 	return correct_vg;
 }
@@ -5547,3 +5653,79 @@ int vg_strip_outdated_historical_lvs(struct volume_group *vg) {
 
 	return 1;
 }
+
+/*
+ * dev is pmem if /sys/dev/block/<major>:<minor>/queue/dax is 1
+ */
+
+int set_cachedev_type(struct physical_volume *pv)
+{
+	FILE *fp;
+	struct device *dev = pv->dev;
+	char path[PATH_MAX];
+	char buffer[64];
+	int is_pmem = 0;
+
+	if (dm_snprintf(path, sizeof(path), "%sdev/block/%d:%d/queue/dax",
+			dm_sysfs_dir(),
+			(int) MAJOR(dev->dev),
+			(int) MINOR(dev->dev)) < 0) {
+		log_warn("Sysfs path for %s dax is too long.", dev_name(dev));
+		return 0;
+	}
+
+	if (!(fp = fopen(path, "r")))
+		return 0;
+
+	if (!fgets(buffer, sizeof(buffer), fp)) {
+		log_warn("Failed to read %s.", path);
+		fclose(fp);
+		return 0;
+	} else if (sscanf(buffer, "%d", &is_pmem) != 1) {
+		log_warn("Failed to parse %s '%s'.", path, buffer);
+		fclose(fp);
+		return 0;
+	}
+
+	fclose(fp);
+
+	if (is_pmem) {
+		log_debug("cachedev %s is pmem", dev_name(dev));
+		dev->is_pmem = 1;
+	}
+
+	return 1;
+}
+
+int set_cachevol_type(struct logical_volume *lv)
+{
+	struct lv_segment *seg;
+	struct physical_volume *pv;
+	uint32_t s;
+	int pmem_devs = 0, other_devs = 0;
+
+	dm_list_iterate_items(seg, &lv->segments) {
+		for (s = 0; s < seg->area_count; s++) {
+			pv = seg_pv(seg, s);
+
+			if (pv->dev->is_pmem) {
+				log_debug("cachevol %s dev %s is pmem.", lv->name, dev_name(pv->dev));
+				pmem_devs++;
+			} else {
+				log_debug("cachevol %s dev %s not pmem.", lv->name, dev_name(pv->dev));
+				other_devs++;
+			}
+		}
+	}
+
+	if (pmem_devs && other_devs) {
+		log_error("Invalid mix of cache device types.");
+		return 0;
+	}
+
+	if (pmem_devs)
+		lv->cachevol_pmem = 1;
+
+	return 1;
+}
+

lib/metadata/metadata.h

@@ -55,7 +55,6 @@
 
 /* May any free extents on this PV be used or must they be left free? */
 
-#define SPINDOWN_LV          	UINT64_C(0x00000010)	/* LV */
 #define BADBLOCK_ON       	UINT64_C(0x00000020)	/* LV */
 //#define VIRTUAL			UINT64_C(0x00010000)	/* LV - internal use only */
 #define PRECOMMITTED		UINT64_C(0x00200000)	/* VG - internal use only */

lib/metadata/pv.c

@@ -237,6 +237,7 @@ char *pv_attr_dup(struct dm_pool *mem, const struct physical_volume *pv)
 	char *repstr;
 	int used = is_used_pv(pv);
 	int duplicate = lvmcache_dev_is_unchosen_duplicate(pv->dev);
+	int cachedev = lvmcache_cachedev_from_pvid(&pv->id) ? 1 : 0;
 
 	if (!(repstr = dm_pool_zalloc(mem, 4))) {
 		log_error("dm_pool_alloc failed");
@@ -248,6 +249,8 @@ char *pv_attr_dup(struct dm_pool *mem, const struct physical_volume *pv)
 	 */
 	if (duplicate)
 		repstr[0] = 'd';
+	else if (cachedev)
+		repstr[0] = 'c';
 	else if (pv->status & ALLOCATABLE_PV)
 		repstr[0] = 'a';
 	else if (used > 0)

lib/metadata/pv.h

@@ -57,8 +57,8 @@ struct physical_volume {
 	unsigned long pe_align;
 	unsigned long pe_align_offset;
 
-        /* This is true whenever the represented PV has a label associated. */
-        uint64_t is_labelled:1;
+	uint64_t is_labelled:1; /* This is true whenever the represented PV has a label associated. */
+	uint64_t is_cachedev:1;
 
         /* NB. label_sector is valid whenever is_labelled is true */
 	uint64_t label_sector;

lib/metadata/pv_manip.c

@@ -115,7 +115,8 @@ static struct pv_segment *_pv_split_segment(struct dm_pool *mem,
 
 	if (peg->lvseg) {
 		peg->pv->pe_alloc_count -= peg_new->len;
-		peg->lvseg->lv->vg->free_count += peg_new->len;
+		if (!pv->is_cachedev)
+			peg->lvseg->lv->vg->free_count += peg_new->len;
 	}
 
 	return peg_new;
@@ -184,7 +185,9 @@ struct pv_segment *assign_peg_to_lvseg(struct physical_volume *pv,
 	peg->lv_area = area_num;
 
 	peg->pv->pe_alloc_count += area_len;
-	peg->lvseg->lv->vg->free_count -= area_len;
+
+	if (!pv->is_cachedev)
+		peg->lvseg->lv->vg->free_count -= area_len;
 
 	return peg;
 }
@@ -195,6 +198,10 @@ int discard_pv_segment(struct pv_segment *peg, uint32_t discard_area_reduction)
 	uint64_t pe_start = peg->pv->pe_start;
 	char uuid[64] __attribute__((aligned(8)));
 
+	/* FIXME: pass in cmd as arg */
+	if (peg->pv->is_cachedev)
+		return 1;
+
 	if (!peg->lvseg) {
 		log_error("discard_pv_segment with unallocated segment: "
 			  "%s PE %" PRIu32, pv_dev_name(peg->pv), peg->pe);
@@ -310,7 +317,8 @@ int release_pv_segment(struct pv_segment *peg, uint32_t area_reduction)
 
 	if (peg->lvseg->area_len == area_reduction) {
 		peg->pv->pe_alloc_count -= area_reduction;
-		peg->lvseg->lv->vg->free_count += area_reduction;
+		if (!peg->pv->is_cachedev)
+			peg->lvseg->lv->vg->free_count += area_reduction;
 
 		peg->lvseg = NULL;
 		peg->lv_area = 0;
@@ -514,8 +522,10 @@ static int _reduce_pv(struct physical_volume *pv, struct volume_group *vg,
 
 	pv->pe_count = new_pe_count;
 
-	vg->extent_count -= (old_pe_count - new_pe_count);
-	vg->free_count -= (old_pe_count - new_pe_count);
+	if (!pv->is_cachedev) {
+		vg->extent_count -= (old_pe_count - new_pe_count);
+		vg->free_count -= (old_pe_count - new_pe_count);
+	}
 
 	return 1;
 }
@@ -542,8 +552,10 @@ static int _extend_pv(struct physical_volume *pv, struct volume_group *vg,
 
 	pv->pe_count = new_pe_count;
 
-	vg->extent_count += (new_pe_count - old_pe_count);
-	vg->free_count += (new_pe_count - old_pe_count);
+	if (!pv->is_cachedev) {
+		vg->extent_count += (new_pe_count - old_pe_count);
+		vg->free_count += (new_pe_count - old_pe_count);
+	}
 
 	return 1;
 }

lib/metadata/segtype.h

@@ -66,6 +66,7 @@ struct dev_manager;
 #define SEG_RAID6_RS_6		(1ULL << 34)
 #define SEG_RAID6_N_6		(1ULL << 35)
 #define SEG_RAID6		SEG_RAID6_ZR
+#define SEG_WRITECACHE		(1ULL << 36)
 
 #define SEG_STRIPED_TARGET	(1ULL << 39)
 #define SEG_LINEAR_TARGET	(1ULL << 40)
@@ -82,6 +83,7 @@ struct dev_manager;
 #define SEG_TYPE_NAME_THIN_POOL		"thin-pool"
 #define SEG_TYPE_NAME_CACHE		"cache"
 #define SEG_TYPE_NAME_CACHE_POOL	"cache-pool"
+#define SEG_TYPE_NAME_WRITECACHE	"writecache"
 #define SEG_TYPE_NAME_ERROR		"error"
 #define SEG_TYPE_NAME_FREE		"free"
 #define SEG_TYPE_NAME_ZERO		"zero"
@@ -114,6 +116,7 @@ struct dev_manager;
 #define segtype_is_striped_target(segtype)	((segtype)->flags & SEG_STRIPED_TARGET ? 1 : 0)
 #define segtype_is_cache(segtype)	((segtype)->flags & SEG_CACHE ? 1 : 0)
 #define segtype_is_cache_pool(segtype)	((segtype)->flags & SEG_CACHE_POOL ? 1 : 0)
+#define segtype_is_writecache(segtype)	((segtype)->flags & SEG_WRITECACHE ? 1 : 0)
 #define segtype_is_mirrored(segtype)	((segtype)->flags & SEG_AREAS_MIRRORED ? 1 : 0)
 #define segtype_is_mirror(segtype)	((segtype)->flags & SEG_MIRROR ? 1 : 0)
 #define segtype_is_pool(segtype)	((segtype)->flags & (SEG_CACHE_POOL | SEG_THIN_POOL)  ? 1 : 0)
@@ -175,6 +178,7 @@ struct dev_manager;
 #define seg_is_striped_target(seg)	segtype_is_striped_target((seg)->segtype)
 #define seg_is_cache(seg)	segtype_is_cache((seg)->segtype)
 #define seg_is_cache_pool(seg)	segtype_is_cache_pool((seg)->segtype)
+#define seg_is_writecache(seg)	segtype_is_writecache((seg)->segtype)
 #define seg_is_used_cache_pool(seg) (seg_is_cache_pool(seg) && (!dm_list_empty(&(seg->lv)->segs_using_this_lv)))
 #define seg_is_linear(seg)	(seg_is_striped(seg) && ((seg)->area_count == 1))
 #define seg_is_mirror(seg)	segtype_is_mirror((seg)->segtype)
@@ -341,6 +345,8 @@ int init_cache_segtypes(struct cmd_context *cmd, struct segtype_library *seglib)
 int init_vdo_segtypes(struct cmd_context *cmd, struct segtype_library *seglib);
 #endif
 
+int init_writecache_segtypes(struct cmd_context *cmd, struct segtype_library *seglib);
+
 #define CACHE_FEATURE_POLICY_MQ			(1U << 0)
 #define CACHE_FEATURE_POLICY_SMQ		(1U << 1)
 #define CACHE_FEATURE_METADATA2			(1U << 2)

lib/metadata/vg.c

@@ -54,6 +54,7 @@ struct volume_group *alloc_vg(const char *pool_name, struct cmd_context *cmd,
 	}
 
 	dm_list_init(&vg->pvs);
+	dm_list_init(&vg->cds);
 	dm_list_init(&vg->pv_write_list);
 	dm_list_init(&vg->lvs);
 	dm_list_init(&vg->historical_lvs);
@@ -664,6 +665,43 @@ char *vg_attr_dup(struct dm_pool *mem, const struct volume_group *vg)
 	return repstr;
 }
 
+static int _vgreduce_cachedev(struct cmd_context *cmd, struct volume_group *vg, struct physical_volume *pv)
+{
+	const char *name = pv_dev_name(pv);
+	struct pv_list *pvl;
+	struct cachedev *cd;
+
+	log_debug("vgreduce_cachedev VG %s PV %s", vg->name, name);
+
+	if (pv_pe_alloc_count(pv)) {
+		log_error("Physical volume \"%s\" still in use", name);
+		return 0;
+	}
+
+	if ((pvl = find_cd_in_vg(vg, name))) {
+		/* delete pv entry from vg->cds */
+		dm_list_del(&pvl->list);
+	}
+
+	if (!vg_write(vg) || !vg_commit(vg)) {
+		log_error("Removal of physical volume \"%s\" from \"%s\" failed", name, vg->name);
+		return 0;
+	}
+
+	if ((cd = lvmcache_cachedev_from_pvid(&pvl->pv->id))) {
+		/* remove ondisk PV header */
+		label_remove(cd->dev);
+		/* delete cd entry from cachedevs */
+		lvmcache_del_cachedev(cd);
+	}
+
+	backup(vg);
+
+	log_print_unless_silent("Removed \"%s\" from volume group \"%s\"", name, vg->name);
+
+	return 1;
+}
+
 int vgreduce_single(struct cmd_context *cmd, struct volume_group *vg,
 			    struct physical_volume *pv, int commit)
 {
@@ -677,6 +715,9 @@ int vgreduce_single(struct cmd_context *cmd, struct volume_group *vg,
 		return r;
 	}
 
+	if (pv->is_cachedev)
+		return _vgreduce_cachedev(cmd, vg, pv);
+
 	log_debug("vgreduce_single VG %s PV %s", vg->name, pv_dev_name(pv));
 
 	if (pv_pe_alloc_count(pv)) {

lib/metadata/vg.h

@@ -75,6 +75,9 @@ struct volume_group {
 	uint32_t pv_count;
 	struct dm_list pvs;
 
+	/* cache devices */
+	struct dm_list cds;
+
 	/*
 	 * List of physical volumes that were used in vgextend but do not carry
 	 * a PV label yet. They need to be pvcreate'd at vg_write time.

lib/report/columns.h

@@ -104,6 +104,8 @@ FIELD(LVS, lv, TIM, "RTime", lvid, 26, lvtimeremoved, lv_time_removed, "Removal
 FIELD(LVS, lv, STR, "Host", lvid, 10, lvhost, lv_host, "Creation host of the LV, if known.", 0)
 FIELD(LVS, lv, STR_LIST, "Modules", lvid, 0, modules, lv_modules, "Kernel device-mapper modules required for this LV.", 0)
 FIELD(LVS, lv, BIN, "Historical", lvid, 0, lvhistorical, lv_historical, "Set if the LV is historical.", 0)
+FIELD(LVS, lv, STR, "Cachevol", lvid, 0, cachevol, cachevol, "An attached cache volume.", 0)
+FIELD(LVS, lv, STR, "Cachevol UUID", lvid, 38, cachevoluuid, cachevol_uuid, "The UUID of an attached cache volume.", 0)
 /*
  * End of LVS type fields
  */

lib/report/properties.c

@@ -388,6 +388,10 @@ GET_LV_STR_PROPERTY_FN(lv_profile, lv_profile_dup(lv->vg->vgmem, lv))
 #define _lv_profile_set prop_not_implemented_set
 GET_LV_STR_PROPERTY_FN(lv_lockargs, lv_lock_args_dup(lv->vg->vgmem, lv))
 #define _lv_lockargs_set prop_not_implemented_set
+GET_LV_STR_PROPERTY_FN(cachevol, lv_cachevol_dup(lv->vg->vgmem, lv))
+#define _cachevol_set prop_not_implemented_set
+GET_LV_STR_PROPERTY_FN(cachevol_uuid, lv_cachevol_uuid_dup(lv->vg->vgmem, lv))
+#define _cachevol_uuid_set prop_not_implemented_set
 
 /* VG */
 GET_VG_STR_PROPERTY_FN(vg_fmt, vg_fmt_dup(vg))

lib/report/report.c

@@ -3765,6 +3765,37 @@ static int _lvhistorical_disp(struct dm_report *rh, struct dm_pool *mem,
 	return _binary_disp(rh, mem, field, lv_is_historical(lv), "historical", private);
 }
 
+static int _do_cachevol_disp(struct dm_report *rh, struct dm_pool *mem,
+			   struct dm_report_field *field,
+			   const void *data, void *private,
+			   int uuid)
+{
+	const struct logical_volume *lv = (const struct logical_volume *) data;
+	struct logical_volume *cachevol_lv = lv_cachevol_lv(lv);
+
+	if (!cachevol_lv)
+		return _field_set_value(field, "", NULL);
+
+	if (uuid)
+		return _uuid_disp(rh, mem, field, &cachevol_lv->lvid.id[1], private);
+
+	return _lvname_disp(rh, mem, field, cachevol_lv, private);
+}
+
+static int _cachevol_disp(struct dm_report *rh, struct dm_pool *mem,
+			    struct dm_report_field *field,
+			    const void *data, void *private)
+{
+	return _do_cachevol_disp(rh, mem, field, data, private, 0);
+}
+
+static int _cachevoluuid_disp(struct dm_report *rh, struct dm_pool *mem,
+			      struct dm_report_field *field,
+			      const void *data, void *private)
+{
+	return _do_cachevol_disp(rh, mem, field, data, private, 1);
+}
+
 /*
  * Macro to generate '_cache_<cache_status_field_name>_disp' reporting function.
  * The 'cache_status_field_name' is field name from struct dm_cache_status.

lib/writecache/writecache.c

@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2013-2016 Red Hat, Inc. All rights reserved.
+ *
+ * This file is part of LVM2.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU Lesser General Public License v.2.1.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "base/memory/zalloc.h"
+#include "lib/misc/lib.h"
+#include "lib/commands/toolcontext.h"
+#include "lib/metadata/segtype.h"
+#include "lib/display/display.h"
+#include "lib/format_text/text_export.h"
+#include "lib/config/config.h"
+#include "lib/datastruct/str_list.h"
+#include "lib/misc/lvm-string.h"
+#include "lib/activate/activate.h"
+#include "lib/metadata/metadata.h"
+#include "lib/metadata/lv_alloc.h"
+#include "lib/config/defaults.h"
+
+static const char _writecache_module[] = "writecache";
+
+#define SEG_LOG_ERROR(t, p...) \
+        log_error(t " segment %s of logical volume %s.", ## p,	\
+                  dm_config_parent_name(sn), seg->lv->name), 0;
+
+static void _writecache_display(const struct lv_segment *seg)
+{
+	/* TODO: lvdisplay segments */
+}
+
+static int _writecache_text_import(struct lv_segment *seg,
+				   const struct dm_config_node *sn,
+				   struct dm_hash_table *pv_hash __attribute__((unused)))
+{
+	struct logical_volume *origin_lv = NULL;
+	struct logical_volume *cachevol;
+	const char *lv_name = NULL;
+	const char *cv_name = NULL;
+
+	if (!dm_config_has_node(sn, "origin"))
+		return SEG_LOG_ERROR("origin not specified in");
+
+	if (!dm_config_get_str(sn, "origin", &lv_name))
+		return SEG_LOG_ERROR("origin must be a string in");
+
+	if (!(origin_lv = find_lv(seg->lv->vg, lv_name)))
+		return SEG_LOG_ERROR("Unknown LV specified for writecache origin %s in", lv_name);
+
+	if (!set_lv_segment_area_lv(seg, 0, origin_lv, 0, 0))
+		return_0;
+
+	if (!dm_config_has_node(sn, "cachevol"))
+		return SEG_LOG_ERROR("cachevol not specified in");
+
+	if (!dm_config_get_str(sn, "cachevol", &cv_name))
+		return SEG_LOG_ERROR("cachevol must be a string in");
+
+	if (!(cachevol = find_lv(seg->lv->vg, cv_name)))
+		return SEG_LOG_ERROR("Unknown logical volume %s specified for cachevol in",
+				     cv_name);
+
+	/* use attach_cachevol() ? */
+
+	seg->origin = origin_lv;
+	seg->cachevol = cachevol;
+	seg->lv->status |= WRITECACHE;
+
+	if (!add_seg_to_segs_using_this_lv(cachevol, seg))
+		return_0;
+
+	return 1;
+}
+
+static int _writecache_text_import_area_count(const struct dm_config_node *sn,
+					      uint32_t *area_count)
+{
+	*area_count = 1;
+
+	return 1;
+}
+
+static int _writecache_text_export(const struct lv_segment *seg,
+				   struct formatter *f)
+{
+	outf(f, "cachevol = \"%s\"", seg->cachevol->name);
+	outf(f, "origin = \"%s\"", seg_lv(seg, 0)->name);
+	return 1;
+}
+
+static void _destroy(struct segment_type *segtype)
+{
+	free((void *) segtype);
+}
+
+#ifdef DEVMAPPER_SUPPORT
+
+static int _target_present(struct cmd_context *cmd,
+			   const struct lv_segment *seg __attribute__((unused)),
+			   unsigned *attributes __attribute__((unused)))
+{
+	static int _writecache_checked = 0;
+	static int _writecache_present = 0;
+
+	if (!activation())
+		return 0;
+
+	if (!_writecache_checked) {
+		_writecache_checked = 1;
+		_writecache_present =  target_present(cmd, TARGET_NAME_WRITECACHE, 0);
+	}
+
+	return _writecache_present;
+}
+
+static int _modules_needed(struct dm_pool *mem,
+			   const struct lv_segment *seg __attribute__((unused)),
+			   struct dm_list *modules)
+{
+	if (!str_list_add(mem, modules, MODULE_NAME_WRITECACHE)) {
+		log_error("String list allocation failed for writecache module.");
+		return 0;
+	}
+
+	return 1;
+}
+#endif /* DEVMAPPER_SUPPORT */
+
+#ifdef DEVMAPPER_SUPPORT
+static int _writecache_add_target_line(struct dev_manager *dm,
+				 struct dm_pool *mem,
+				 struct cmd_context *cmd __attribute__((unused)),
+				 void **target_state __attribute__((unused)),
+				 struct lv_segment *seg,
+				 const struct lv_activate_opts *laopts __attribute__((unused)),
+				 struct dm_tree_node *node, uint64_t len,
+				 uint32_t *pvmove_mirror_count __attribute__((unused)))
+{
+	char *origin_uuid;
+	char *cache_uuid;
+	int pmem;
+
+	if (!seg_is_writecache(seg)) {
+		log_error(INTERNAL_ERROR "Passed segment is not writecache.");
+		return 0;
+	}
+
+	if (!seg->cachevol) {
+		log_error(INTERNAL_ERROR "Passed segment has no cachevol.");
+		return 0;
+	}
+
+	pmem = seg->cachevol->cachevol_pmem;
+
+	if (!(origin_uuid = build_dm_uuid(mem, seg_lv(seg, 0), NULL)))
+		return_0;
+
+	if (!(cache_uuid = build_dm_uuid(mem, seg->cachevol, NULL)))
+		return_0;
+
+	if (!dm_tree_node_add_writecache_target(node, len, origin_uuid, cache_uuid, pmem))
+		return_0;
+
+	return 1;
+}
+#endif /* DEVMAPPER_SUPPORT */
+
+static struct segtype_handler _writecache_ops = {
+	.display = _writecache_display,
+	.text_import = _writecache_text_import,
+	.text_import_area_count = _writecache_text_import_area_count,
+	.text_export = _writecache_text_export,
+#ifdef DEVMAPPER_SUPPORT
+	.add_target_line = _writecache_add_target_line,
+	.target_present = _target_present,
+	.modules_needed = _modules_needed,
+#endif
+	.destroy = _destroy,
+};
+
+int init_writecache_segtypes(struct cmd_context *cmd,
+			struct segtype_library *seglib)
+{
+	struct segment_type *segtype = zalloc(sizeof(*segtype));
+
+	if (!segtype) {
+		log_error("Failed to allocate memory for cache_pool segtype");
+		return 0;
+	}
+
+	segtype->name = SEG_TYPE_NAME_WRITECACHE;
+	segtype->flags = SEG_WRITECACHE;
+	segtype->ops = &_writecache_ops;
+
+	if (!lvm_register_segtype(seglib, segtype))
+		return_0;
+	log_very_verbose("Initialised segtype: %s", segtype->name);
+
+	return 1;
+}

tools/args.h

@@ -102,6 +102,18 @@ arg(cache_long_ARG, '\0', "cache", 0, 0, 0,
     "#lvscan\n"
     "This option is no longer used.\n")
 
+arg(cachedev_ARG, '\0', "cachedev", 0, 0, 0,
+    "The specified device is a cache device, not a normal PV.\n")
+
+arg(cachevol_ARG, '\0', "cachevol", 0, 0, 0,
+    "Operate on cache volumes and cache devices.\n")
+
+arg(cachevolattach_ARG, '\0', "cachevolattach", lv_VAL, 0, 0,
+    "Attach a cache volume to an LV to perform caching of the LV.\n")
+
+arg(cachevoldetach_ARG, '\0', "cachevoldetach", lv_VAL, 0, 0,
+    "Detach a cache volume from an LV.\n")
+
 arg(cachemetadataformat_ARG, '\0', "cachemetadataformat", cachemetadataformat_VAL, 0, 0,
     "Specifies the cache metadata format used by cache target.\n")
 

tools/command-lines.in

@@ -732,6 +732,17 @@ DESC: Poll LV to continue conversion (also see --startpoll)
 DESC: or waits till conversion/mirror syncing is finished
 FLAGS: SECONDARY_SYNTAX
 
+----
+
+lvconvert --cachevolattach LV LV_linear_striped
+OO: --type writecache
+ID: lvconvert_cachevol_attach
+DESC: Attach a cache volume to an LV.
+
+lvconvert --cachevoldetach LV LV_writecache
+ID: lvconvert_cachevol_detach
+DESC: Detach a cache volume from an LV.
+
 ---
 
 # --extents is not specified; it's an automatic alternative for --size
@@ -1241,6 +1252,13 @@ DESC: to type cache after creating a new cache pool LV to use
 DESC: (use lvconvert).
 FLAGS: SECONDARY_SYNTAX
 
+lvcreate --cachevol --size SizeMB VG
+OO: --type linear, OO_LVCREATE
+OP: PV ...
+IO: --mirrors 0, --stripes 1
+ID: lvcreate_cachevol
+DESC: Create a cache volume on cache devices.
+
 ---
 
 lvdisplay
@@ -1646,6 +1664,10 @@ OO: --autobackup Bool,
 --reportformat ReportFmt, --restoremissing
 ID: vgextend_general
 
+vgextend --cachedev VG PV ...
+OO: --reportformat ReportFmt
+ID: vgextend_cachedev
+
 ---
 
 OO_VGIMPORT: --force, --reportformat ReportFmt

tools/command.c

@@ -137,6 +137,7 @@ static inline int configtype_arg(struct cmd_context *cmd __attribute__((unused))
 #define DISALLOW_TAG_ARGS        0x00000800
 #define GET_VGNAME_FROM_OPTIONS  0x00001000
 #define CAN_USE_ONE_SCAN	 0x00002000
+#define ENABLE_CACHE_DEVS	 0x00004000
 
 /* create foo_CMD enums for command def ID's in command-lines.in */
 

tools/commands.h

@@ -145,7 +145,7 @@ xx(pvremove,
 
 xx(pvs,
    "Display information about physical volumes",
-   PERMITTED_READ_ONLY | ALL_VGS_IS_DEFAULT | ENABLE_ALL_DEVS | ENABLE_DUPLICATE_DEVS | LOCKD_VG_SH | CAN_USE_ONE_SCAN)
+   PERMITTED_READ_ONLY | ALL_VGS_IS_DEFAULT | ENABLE_ALL_DEVS | ENABLE_CACHE_DEVS | ENABLE_DUPLICATE_DEVS | LOCKD_VG_SH | CAN_USE_ONE_SCAN)
 
 xx(pvscan,
    "List all physical volumes",
@@ -216,7 +216,7 @@ xx(vgmknodes,
 
 xx(vgreduce,
    "Remove physical volume(s) from a volume group",
-   0)
+   ENABLE_CACHE_DEVS)
 
 xx(vgremove,
    "Remove volume group(s)",

tools/lv_types.h

@@ -33,5 +33,6 @@ lvt(raid6_LVT, "raid6", NULL)
 lvt(raid10_LVT, "raid10", NULL)
 lvt(error_LVT, "error", NULL)
 lvt(zero_LVT, "zero", NULL)
+lvt(writecache_LVT, "writecache", NULL)
 lvt(LVT_COUNT, "", NULL)
 

tools/lvconvert.c

@@ -16,6 +16,7 @@
 
 #include "lib/lvmpolld/polldaemon.h"
 #include "lib/metadata/lv_alloc.h"
+#include "lib/metadata/metadata.h"
 #include "lvconvert_poll.h"
 
 #define MAX_PDATA_ARGS	10	/* Max number of accepted args for d-m-p-d tools */
@@ -4858,6 +4859,266 @@ int lvconvert_to_vdopool_param_cmd(struct cmd_context *cmd, int argc, char **arg
 			       NULL, NULL, &_lvconvert_to_vdopool_single);
 }
 
+static int _cachevol_zero(struct cmd_context *cmd,
+			 struct logical_volume *lv)
+{
+	struct device *dev;
+	char name[PATH_MAX];
+	int ret = 0;
+
+	if (!activate_lv(cmd, lv)) {
+		log_error("Failed to activate LV %s for zeroing.", lv->name);
+		return 0;
+	}
+
+	sync_local_dev_names(cmd);
+
+	if (dm_snprintf(name, sizeof(name), "%s%s/%s",
+			cmd->dev_dir, lv->vg->name, lv->name) < 0) {
+		log_error("Name too long - device not cleared (%s)", lv->name);
+		goto out;
+	}
+
+	if (!(dev = dev_cache_get(cmd, name, NULL))) {
+		log_error("%s: not found: device not zeroed", name);
+		goto out;
+	}
+
+	if (!label_scan_open(dev)) {
+		log_error("Failed to open %s/%s for zeroing.", lv->vg->name, lv->name);
+		goto out;
+	}
+
+	if (!dev_write_zeros(dev, UINT64_C(0), (size_t) 1 << SECTOR_SHIFT))
+		goto_out;
+
+	label_scan_invalidate(dev);
+
+	ret = 1;
+out:
+	if (!deactivate_lv(cmd, lv)) {
+		log_error("Failed to deactivate LV %s for zeroing.", lv->name);
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static struct logical_volume *_lv_writecache_create(struct cmd_context *cmd,
+					    struct logical_volume *lv,
+					    struct logical_volume *cv)
+{
+	const struct segment_type *segtype;
+	struct lv_segment *seg;
+
+	/* TODO: verify type/properties of cv */
+
+	if (!(segtype = get_segtype_from_string(cmd, SEG_TYPE_NAME_WRITECACHE)))
+		return_NULL;
+
+	if (!insert_layer_for_lv(cmd, lv, WRITECACHE, "_wcorig"))
+		return_NULL;
+
+	lv_set_hidden(cv);
+
+	seg = first_seg(lv);
+	seg->segtype = segtype;
+
+	seg->cachevol = cv;
+
+	add_seg_to_segs_using_this_lv(cv, seg);
+
+	return lv;
+}
+
+static int _lvconvert_cachevol_attach_single(struct cmd_context *cmd,
+					struct logical_volume *lv,
+					struct processing_handle *handle)
+{
+	struct logical_volume *new_lv;
+	struct logical_volume *cv;
+	const char *cvname;
+
+	cvname = arg_str_value(cmd, cachevolattach_ARG, "");
+
+	if (!(cv = find_lv(lv->vg, cvname))) {
+		log_error("Cache volume %s not found.", cvname);
+		goto bad;
+	}
+
+	if (!lv_is_cachevol(cv)) {
+		log_error("LV %s is not a cache volume.", cvname);
+		goto bad;
+	}
+
+	if (lv_info(cmd, lv, 1, NULL, 0, 0)) {
+		log_error("LV must be inactive to attach a cachevol.");
+		return 0;
+	}
+
+	/* CV shouldn't generally be active by itself, but just in case. */
+	if (lv_info(cmd, cv, 1, NULL, 0, 0)) {
+		log_error("cachevol must be inactive to attach.");
+		return 0;
+	}
+
+	if (!_cachevol_zero(cmd, cv)) {
+		log_error("cachevol %s could not be zeroed.", cv->name);
+		return 0;
+	}
+
+	/* If LV is inactive here, ensure it's not active elsewhere. */
+	if (!lockd_lv(cmd, lv, "ex", 0))
+		goto_bad;
+
+	if (!archive(lv->vg))
+		goto_bad;
+
+	/* Changes the vg struct to match the desired state. */
+
+	if (!(new_lv = _lv_writecache_create(cmd, lv, cv)))
+		goto_bad;
+
+	/*
+	 * vg_write(), suspend_lv(), vg_commit(), resume_lv(),
+	 * where the old LV is suspended and the new LV is resumed.
+	 */
+
+	if (!lv_update_and_reload(new_lv))
+		goto_bad;
+
+	log_print_unless_silent("Logical volume %s now has write cache.",
+				display_lvname(new_lv));
+	return ECMD_PROCESSED;
+bad:
+	return ECMD_FAILED;
+
+}
+
+static int lv_cachevol_detach(struct cmd_context *cmd,
+			      struct logical_volume *lv)
+{
+	struct lv_segment *seg = first_seg(lv);
+	struct logical_volume *origin;
+
+	if (lv_info(cmd, lv, 1, NULL, 0, 0)) {
+		log_error("LV must be inactive to detach writecache.");
+		return 0;
+	}
+
+	/* LV is inactive */
+
+	if (!seg_is_writecache(seg)) {
+		log_error("LV %s segment is not writecache", lv->name);
+		return 0;
+	}
+
+	if (!seg->cachevol) {
+		log_error("LV %s writecache segment has no cachevol", lv->name);
+		return 0;
+	}
+
+	if (!(origin = seg_lv(seg, 0))) {
+		log_error("LV %s writecache segment has no origin", lv->name);
+		return 0;
+	}
+
+	if (!remove_seg_from_segs_using_this_lv(seg->cachevol, seg))
+		return_0;
+
+	lv_set_visible(seg->cachevol);
+
+	lv->status &= ~WRITECACHE;
+	seg->cachevol = NULL;
+
+	if (!remove_layer_from_lv(lv, origin))
+		return_0;
+
+	if (!lv_remove(origin))
+		return_0;
+
+	return 1;
+}
+
+static int _lvconvert_cachevol_detach_single(struct cmd_context *cmd,
+					struct logical_volume *lv,
+					struct processing_handle *handle)
+{
+	struct logical_volume *cv;
+	const char *cvname;
+
+	cvname = arg_str_value(cmd, cachevoldetach_ARG, "");
+
+	if (!lv_is_writecache(lv)) {
+		log_error("LV %s does not have an attached cachevol.", lv->name);
+		goto bad;
+	}
+
+	if (!(cv = find_lv(lv->vg, cvname))) {
+		log_error("Cache volume %s not found.", cvname);
+		goto bad;
+	}
+
+	if (!lv_is_cachevol(cv)) {
+		log_error("LV %s is not a cache volume.", cvname);
+		goto bad;
+	}
+
+	if (first_seg(lv)->cachevol != cv) {
+		log_error("LV %s cachevol does not match specified %s.", lv->name, cvname);
+		goto bad;
+	}
+
+	/* If LV is inactive here, ensure it's not active elsewhere. */
+	if (!lockd_lv(cmd, lv, "ex", 0))
+		goto_bad;
+
+	if (!archive(lv->vg))
+		goto_bad;
+
+	if (!lv_cachevol_detach(cmd, lv))
+		return_0;
+
+	if (!vg_write(lv->vg) || !vg_commit(lv->vg))
+		return_0;
+
+	backup(lv->vg);
+
+	log_print_unless_silent("Logical volume %s write cache has been removed.",
+				display_lvname(lv));
+	return ECMD_PROCESSED;
+bad:
+	return ECMD_FAILED;
+
+}
+
+int lvconvert_cachevol_cmd(struct cmd_context *cmd, int argc, char **argv)
+{
+	struct processing_handle *handle;
+	struct lvconvert_result lr = { 0 };
+	int attach = 0;
+	int ret;
+
+	if (cmd->command->command_enum == lvconvert_cachevol_attach_CMD)
+		attach = 1;
+
+	if (!(handle = init_processing_handle(cmd, NULL))) {
+		log_error("Failed to initialize processing handle.");
+		return ECMD_FAILED;
+	}
+
+	handle->custom_handle = &lr;
+
+	cmd->cname->flags &= ~GET_VGNAME_FROM_OPTIONS;
+
+	ret = process_each_lv(cmd, cmd->position_argc, cmd->position_argv, NULL, NULL, READ_FOR_UPDATE, handle, NULL,
+			      attach ? &_lvconvert_cachevol_attach_single : &_lvconvert_cachevol_detach_single);
+
+	destroy_processing_handle(cmd, handle);
+
+	return ret;
+}
+
 /*
  * All lvconvert command defs have their own function,
  * so the generic function name is unused.

tools/lvcreate.c

@@ -1770,3 +1770,119 @@ int lvcreate(struct cmd_context *cmd, int argc, char **argv)
 	destroy_processing_handle(cmd, handle);
 	return ret;
 }
+
+static int _lvcreate_cachevol_single(struct cmd_context *cmd, const char *vg_name,
+				     struct volume_group *vg, struct processing_handle *handle)
+{
+	struct processing_params *pp = (struct processing_params *) handle->custom_handle;
+	struct lvcreate_params *lp = pp->lp;
+	struct lvcreate_cmdline_params *lcp = pp->lcp;
+	struct logical_volume *lv;
+	struct dm_list *use_pvh;
+	int ret = ECMD_FAILED;
+
+	if (arg_is_set(cmd, extents_ARG))
+		lp->extents = arg_uint_value(cmd, extents_ARG, 0);
+	else if (arg_is_set(cmd, size_ARG)) {
+		lcp->size = arg_uint64_value(cmd, size_ARG, UINT64_C(0));
+		lp->extents = extents_from_size(cmd, lcp->size, vg->extent_size);
+	}
+
+	if (!lp->extents) {
+		log_error("The size of the LV is zero extents.");
+		goto out;
+	}
+
+	lp->stripes = 1;
+
+	if (!_check_zero_parameters(cmd, lp))
+		return_0;
+
+	/*
+	 * cachevols can be allocated from PVs by specifying the PV device
+	 * name(s).  Otherwise, cachevols are only allocated from CDs.
+	 */
+	if (cmd->position_argc > 1) {
+		/* First pos arg is required LV, remaining are optional CDs. */
+		if (!(use_pvh = create_cd_list(cmd->mem, vg, cmd->position_argc - 1, cmd->position_argv + 1))) {
+			if (!(use_pvh = create_pv_list(cmd->mem, vg, cmd->position_argc - 1, cmd->position_argv + 1, 1)))
+				return_ECMD_FAILED;
+		}
+		lcp->pv_count = cmd->position_argc - 1;
+	} else
+		use_pvh = &vg->cds;
+
+	lp->pvh = use_pvh;
+
+	if (dm_list_empty(lp->pvh)) {
+		log_error("No cache devices found to create a cache volume.");
+		log_error("Use vgextend --cachedev VG <cache devices>.");
+		goto out;
+	}
+
+	lp->cachevol = 1; /* Tells lvcreate to set lv status flag CACHEVOL. */
+
+	if (!(lv = lv_create_single(vg, lp)))
+		goto_out;
+
+	if (!set_cachevol_type(lv))
+		goto_out;
+
+	ret = ECMD_PROCESSED;
+out:
+	return ret;
+}
+
+int lvcreate_cachevol_cmd(struct cmd_context *cmd, int argc, char **argv)
+{
+	struct processing_handle *handle = NULL;
+	struct processing_params pp;
+	struct lvcreate_params lp = {
+		.major = -1,
+		.minor = -1,
+	};
+	struct lvcreate_cmdline_params lcp = { 0 };
+	const char *vg_name;
+	int ret;
+
+	vg_name = skip_dev_dir(cmd, argv[0], NULL);
+	argc--;
+	argv++;
+
+	dm_list_init(&lp.tags);
+	lp.target_attr = ~0;
+	lp.yes = arg_count(cmd, yes_ARG);
+	lp.force = (force_t) arg_count(cmd, force_ARG);
+	lp.segtype = get_segtype_from_string(cmd, SEG_TYPE_NAME_STRIPED);
+	lp.alloc = ALLOC_CONTIGUOUS;
+	lp.activate = CHANGE_AN;
+	lp.permission = LVM_READ | LVM_WRITE;
+
+	lp.lv_name = arg_str_value(cmd, name_ARG, NULL);
+	if (!validate_restricted_lvname_param(cmd, &lp.vg_name, &lp.lv_name))
+		return_0;
+
+	pp.lp = &lp;
+	pp.lcp = &lcp;
+
+	if (!(handle = init_processing_handle(cmd, NULL))) {
+		log_error("Failed to initialize processing handle.");
+		return ECMD_FAILED;
+	}
+
+	if (!(handle = init_processing_handle(cmd, NULL))) {
+		log_error("Failed to initialize processing handle.");
+		return ECMD_FAILED;
+	}
+
+	handle->custom_handle = &pp;
+
+	/* eventually set this in commands.h when all lvcreates use it */
+	cmd->cname->flags |= GET_VGNAME_FROM_OPTIONS;
+
+	ret = process_each_vg(cmd, 0, NULL, vg_name, NULL, READ_FOR_UPDATE, 0,
+			      handle, &_lvcreate_cachevol_single);
+
+	destroy_processing_handle(cmd, handle);
+	return ret;
+}

tools/lvmcmdline.c

@@ -147,8 +147,13 @@ static const struct command_function _command_functions[CMD_COUNT] = {
 	{ lvconvert_to_vdopool_CMD, lvconvert_to_vdopool_cmd },
 	{ lvconvert_to_vdopool_param_CMD, lvconvert_to_vdopool_param_cmd },
 
+	{ lvconvert_cachevol_attach_CMD, lvconvert_cachevol_cmd },
+	{ lvconvert_cachevol_detach_CMD, lvconvert_cachevol_cmd },
+
 	{ pvscan_display_CMD, pvscan_display_cmd },
 	{ pvscan_cache_CMD, pvscan_cache_cmd },
+
+	{ lvcreate_cachevol_CMD, lvcreate_cachevol_cmd },
 };
 
 

tools/pvscan.c

@@ -376,13 +376,24 @@ static int _online_pv_found(struct cmd_context *cmd,
 			dev_args_in_vg = 1;
 	}
 
+	dm_list_iterate_items(pvl, &vg->cds) {
+		if (!_online_pvid_file_exists((const char *)&pvl->pv->id.uuid))
+			pvids_not_online++;
+
+		/* Check if one of the devs on the command line is in this VG. */
+		if (dev_args && dev_in_device_list(pvl->pv->dev, dev_args))
+			dev_args_in_vg = 1;
+	}
+
 	/*
 	 * Return if we did not find an online file for one of the PVIDs
 	 * in the VG, which means the VG is not yet complete.
 	 */
 
-	if (pvids_not_online)
+	if (pvids_not_online) {
+		log_debug("missing %d PVs in VG %s.", pvids_not_online, vg->name);
 		return 1;
+	}
 
 	/*
 	 * When all PVIDs from the VG are online, then add vgname to

tools/toollib.c

@@ -15,6 +15,9 @@
 
 #include "tools.h"
 #include "lib/format_text/format-text.h"
+#include "lib/format_text/layout.h"
+#include "lib/mm/xlate.h"
+#include "lib/misc/crc.h"
 
 #include <sys/stat.h>
 #include <signal.h>
@@ -629,16 +632,23 @@ static int _create_pv_entry(struct dm_pool *mem, struct pv_list *pvl,
 	return 1;
 }
 
-struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int argc,
-			    char **argv, int allocatable_only)
+static struct dm_list *_create_pv_list(struct dm_pool *mem, struct volume_group *vg,
+				       int argc, char **argv,
+				       int allocatable_only, int cachedev)
 {
 	struct dm_list *r;
+	struct dm_list *pvlist;
 	struct pv_list *pvl;
 	struct dm_list tagsl, arg_pvnames;
 	char *pvname = NULL;
 	char *colon, *at_sign, *tagname;
 	int i;
 
+	if (cachedev)
+		pvlist = &vg->cds;
+	else
+		pvlist = &vg->pvs;
+
 	/* Build up list of PVs */
 	if (!(r = dm_pool_alloc(mem, sizeof(*r)))) {
 		log_error("Allocation of list failed.");
@@ -658,7 +668,7 @@ struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int
 				log_error("Skipping invalid tag %s.", tagname);
 				continue;
 			}
-			dm_list_iterate_items(pvl, &vg->pvs) {
+			dm_list_iterate_items(pvl, pvlist) {
 				if (str_list_match_item(&pvl->pv->tags,
 							tagname)) {
 					if (!_create_pv_entry(mem, pvl, NULL,
@@ -678,7 +688,12 @@ struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int
 			return NULL;
 		}
 
-		if (!(pvl = find_pv_in_vg(vg, pvname))) {
+		if (cachedev)
+			pvl = find_cd_in_vg(vg, pvname);
+		else
+			pvl = find_pv_in_vg(vg, pvname);
+
+		if (!pvl) {
 			log_error("Physical Volume \"%s\" not found in "
 				  "Volume Group \"%s\".", pvname, vg->name);
 			return NULL;
@@ -693,6 +708,17 @@ struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int
 	return dm_list_empty(r) ? NULL : r;
 }
 
+struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int argc,
+			    char **argv, int allocatable_only)
+{
+	return _create_pv_list(mem, vg, argc, argv, allocatable_only, 0);
+}
+
+struct dm_list *create_cd_list(struct dm_pool *mem, struct volume_group *vg, int argc, char **argv)
+{
+	return _create_pv_list(mem, vg, argc, argv, 0, 1);
+}
+
 struct dm_list *clone_pv_list(struct dm_pool *mem, struct dm_list *pvsl)
 {
 	struct dm_list *r;
@@ -1003,6 +1029,11 @@ int lv_change_activate(struct cmd_context *cmd, struct logical_volume *lv,
 		 */
 	}
 
+	if (lv_is_cachevol(lv)) {
+		log_verbose("Skipping cachevol %s.", display_lvname(lv));
+		return 1;
+	}
+
 	if (lv_is_merging_origin(lv)) {
 		/*
 		 * For merging origin, its snapshot must be inactive.
@@ -2561,6 +2592,8 @@ static int _lv_is_type(struct cmd_context *cmd, struct logical_volume *lv, int l
 		return seg_is_any_raid6(seg);
 	case raid10_LVT:
 		return seg_is_raid10(seg);
+	case writecache_LVT:
+		return seg_is_writecache(seg);
 	case error_LVT:
 		return !strcmp(seg->segtype->name, SEG_TYPE_NAME_ERROR);
 	case zero_LVT:
@@ -2617,6 +2650,8 @@ int get_lvt_enum(struct logical_volume *lv)
 		return raid6_LVT;
 	if (seg_is_raid10(seg))
 		return raid10_LVT;
+	if (seg_is_writecache(seg))
+		return writecache_LVT;
 
 	if (!strcmp(seg->segtype->name, SEG_TYPE_NAME_ERROR))
 		return error_LVT;
@@ -2739,8 +2774,13 @@ static int _check_lv_types(struct cmd_context *cmd, struct logical_volume *lv, i
 	if (!ret) {
 		int lvt_enum = get_lvt_enum(lv);
 		struct lv_type *type = get_lv_type(lvt_enum);
-		log_warn("Command on LV %s does not accept LV type %s.",
-			 display_lvname(lv), type ? type->name : "unknown");
+		if (!type) {
+			log_warn("Command on LV %s does not accept LV type unknown (%d).",
+				 display_lvname(lv), lvt_enum);
+		} else {
+			log_warn("Command on LV %s does not accept LV type %s.",
+				 display_lvname(lv), type->name);
+		}
 	}
 
 	return ret;
@@ -4155,6 +4195,7 @@ static int _process_duplicate_pvs(struct cmd_context *cmd,
 
 static int _process_pvs_in_vg(struct cmd_context *cmd,
 			      struct volume_group *vg,
+			      struct dm_list *pv_list,
 			      struct dm_list *all_devices,
 			      struct dm_list *arg_devices,
 			      struct dm_list *arg_tags,
@@ -4197,7 +4238,7 @@ static int _process_pvs_in_vg(struct cmd_context *cmd,
 	if (!is_orphan_vg(vg->name))
 		log_set_report_object_group_and_group_id(vg->name, vg_uuid);
 
-	dm_list_iterate_items(pvl, &vg->pvs) {
+	dm_list_iterate_items(pvl, pv_list) {
 		pv = pvl->pv;
 		pv_name = pv_dev_name(pv);
 		pv_uuid[0]='\0';
@@ -4360,11 +4401,20 @@ static int _process_pvs_in_vgs(struct cmd_context *cmd, uint32_t read_flags,
 		 * vg->pvs entries from devices list.
 		 */
 		
-		ret = _process_pvs_in_vg(cmd, vg, all_devices, arg_devices, arg_tags,
+		ret = _process_pvs_in_vg(cmd, vg, &vg->pvs, all_devices, arg_devices, arg_tags,
 					 process_all_pvs, process_all_devices, skip,
 					 handle, process_single_pv);
 		if (ret != ECMD_PROCESSED)
 			stack;
+
+		if (cmd->cname->flags & ENABLE_CACHE_DEVS) {
+			ret = _process_pvs_in_vg(cmd, vg, &vg->cds, all_devices, arg_devices, arg_tags,
+						 process_all_pvs, process_all_devices, skip,
+						 handle, process_single_pv);
+			if (ret != ECMD_PROCESSED)
+				stack;
+		}
+
 		report_log_ret_code(ret);
 		if (ret > ret_max)
 			ret_max = ret;
@@ -5035,6 +5085,7 @@ static int _pvcreate_check_single(struct cmd_context *cmd,
 		log_debug("Found pvcreate arg %s: pv is used in %s.", pd->name, vg->name);
 		pd->is_vg_pv = 1;
 		pd->vg_name = dm_pool_strdup(cmd->mem, vg->name);
+
 	} else if (vg && is_orphan_vg(vg->name)) {
 		if (is_used_pv(pv)) {
 			/* Device is used in an unknown VG. */
@@ -5047,6 +5098,12 @@ static int _pvcreate_check_single(struct cmd_context *cmd,
 		}
 
 		pp->orphan_vg_name = FMT_TEXT_ORPHAN_VG_NAME;
+
+	} else if (lvmcache_cachedev_from_pvid((const struct id *)pv->dev->pvid)) {
+		log_error("Cannot use cache device %s.", pd->name);
+		dm_list_move(&pp->arg_fail, &pd->list);
+		return 1;
+
 	} else {
 		log_debug("Found pvcreate arg %s: device is not a PV.", pd->name);
 		/* Device is not a PV. */
@@ -5210,6 +5267,7 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 	struct pvcreate_params *pp = (struct pvcreate_params *) handle->custom_handle;
 	struct pvcreate_device *pd;
 	struct pvcreate_prompt *prompt;
+	struct cachedev *cd;
 	int found = 0;
 
 	if (!pv->dev)
@@ -5235,12 +5293,13 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 	log_debug("Checking device %s for pvremove %.32s.",
 		  pv_dev_name(pv), pv->dev->pvid[0] ? pv->dev->pvid : "");
 
+	cd = lvmcache_cachedev_from_pvid((const struct id *)pv->dev->pvid);
 
 	/*
 	 * Is there a pv here already?
 	 * If not, this is an error unless you used -f.
 	 */
-	if (!lvmcache_has_dev_info(pv->dev)) {
+	if (!lvmcache_has_dev_info(pv->dev) && !cd) {
 		if (pp->force) {
 			dm_list_move(&pp->arg_process, &pd->list);
 			return 1;
@@ -5258,6 +5317,16 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 		/* Device is not a PV. */
 		log_debug("Found pvremove arg %s: device is not a PV.", pd->name);
 
+	} else if (cd) {
+		/* Device is a cachedev PV used in a VG. */
+		log_debug("Found pvremove arg %s: pv is cache device.", pd->name);
+		if (cd->vg_name) {
+			pd->is_vg_pv = 1;
+			pd->vg_name = strdup(cd->vg_name);
+		} else {
+			pd->is_used_unknown_pv = 1;
+		}
+
 	} else if (vg && !is_orphan_vg(vg->name)) {
 		/* Device is a PV used in a VG. */
 		log_debug("Found pvremove arg %s: pv is used in %s.", pd->name, vg->name);
@@ -5276,6 +5345,7 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 		}
 
 		pp->orphan_vg_name = FMT_TEXT_ORPHAN_VG_NAME;
+
 	} else {
 		/* FIXME: is it possible to reach here? */
 		log_debug("Found pvremove arg %s: device is not a PV.", pd->name);
@@ -5315,7 +5385,7 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 	if (pd->is_used_unknown_pv)
 		prompt->vg_name_unknown = 1;
 	else
-		prompt->vg_name = dm_pool_strdup(cmd->mem, vg->name);
+		prompt->vg_name = dm_pool_strdup(cmd->mem, pd->vg_name);
 	prompt->type |= PROMPT_PVREMOVE_PV_IN_VG;
 	dm_list_add(&pp->prompts, &prompt->list);
 
@@ -5325,6 +5395,85 @@ static int _pvremove_check_single(struct cmd_context *cmd,
 	return 1;
 }
 
+static struct physical_volume *_cachedev_pv_create(struct cmd_context *cmd, struct device *dev)
+{
+	char buf[LABEL_SIZE];
+	struct physical_volume *pv;
+	struct label_header *lh;
+	struct pv_header *pvhdr;
+	struct pv_header_extension *pvext;
+	uint32_t pvhdr_len;
+	uint32_t crc;
+
+	if (!(pv = dm_pool_zalloc(cmd->mem, sizeof(*pv))))
+		return_NULL;
+
+	pv->dev = dev;
+	dm_list_init(&pv->tags);
+	dm_list_init(&pv->segments);
+
+	if (!id_create(&pv->id))
+		return_NULL;
+
+	if (!dev_get_size(dev, &pv->size)) {
+		log_error("%s: Couldn't get size.", dev_name(dev));
+		return NULL;
+	}
+
+	memset(buf, 0, LABEL_SIZE);
+
+	/*
+	 * Set label_header fields (except crc).
+	 * Set pv_header fields.
+	 * Set pv_header_extension fields.
+	 * Calculate crc.
+	 * Set label_header crc field.
+	 *
+	 * The label_header crc is calculated on data from just after the lh
+	 * crc field (starting with lh.offset) to the end of the label sector,
+	 * which includes the pv_header and pv_header_extension.  So, the
+	 * pv_header and pv_header_extenstion fields need to be set before the
+	 * lh crc is calculated and set in the lh.
+	 *
+	 * The pv_header needs to be followed by two empty disk_locn structs,
+	 * the first terminates the non-existant data areas list, and the
+	 * second terminates the non-existant metadata areas list.
+	 */
+
+	lh = (struct label_header *)buf;
+	strncpy((char *)lh->id, LABEL_ID, sizeof(lh->id));
+	lh->sector_xl = xlate64(DEFAULT_LABELSECTOR);
+	lh->offset_xl = xlate32(sizeof(struct label_header));
+	strncpy((char *)lh->type, LVM2_LABEL, sizeof(lh->type));
+
+	pvhdr = (struct pv_header *) ((char *)buf + sizeof(struct label_header));
+	pvhdr->device_size_xl = xlate64(pv->size);
+	memcpy(pvhdr->pv_uuid, &pv->id, sizeof(struct id));
+
+	/* Two empty disk_locn structs follow the struct pv_header. */
+	pvhdr_len = sizeof(struct pv_header) + (2 * sizeof(struct disk_locn));
+
+	pvext = (struct pv_header_extension *) ((char *)pvhdr + pvhdr_len);
+	pvext->version = xlate32(PV_HEADER_EXTENSION_VSN);
+	pvext->flags = xlate32(PV_EXT_USED);
+
+	crc = calc_crc(INITIAL_CRC,
+		       (uint8_t *)&lh->offset_xl,
+		       LABEL_SIZE - ((uint8_t *) &lh->offset_xl - (uint8_t *) lh));
+
+	lh->crc_xl = xlate32(crc);
+
+	if (!dev_write_bytes(dev, DEFAULT_LABELSECTOR << SECTOR_SHIFT, LABEL_SIZE, buf)) {
+		log_debug_devs("Failed to write label to %s", dev_name(dev));
+		return_NULL;
+	}
+
+	/* Should we add an entry to the cachdevs list as would happen if
+	   this device was scanned? */
+
+	return pv;
+}
+
 /*
  * This can be used by pvcreate, vgcreate and vgextend to create PVs.  The
  * callers need to set up the pvcreate_each_params structure based on command
@@ -5428,7 +5577,7 @@ int pvcreate_each_device(struct cmd_context *cmd,
 	 * If it's added to arg_process but needs a prompt or force option, then
 	 * a corresponding prompt entry is added to pp->prompts.
 	 */
-	process_each_pv(cmd, 0, NULL, NULL, 1, PROCESS_SKIP_SCAN | PROCESS_SKIP_ORPHAN_LOCK,
+	process_each_pv(cmd, 0, NULL, NULL, 1, PROCESS_SKIP_SCAN | PROCESS_SKIP_ORPHAN_LOCK | ENABLE_CACHE_DEVS,
 			handle, pp->is_remove ? _pvremove_check_single : _pvcreate_check_single);
 
 	/*
@@ -5576,7 +5725,7 @@ int pvcreate_each_device(struct cmd_context *cmd,
 	 */
 	dm_list_splice(&pp->arg_confirm, &pp->arg_process);
 
-	process_each_pv(cmd, 0, NULL, NULL, 1, PROCESS_SKIP_SCAN | PROCESS_SKIP_ORPHAN_LOCK,
+	process_each_pv(cmd, 0, NULL, NULL, 1, PROCESS_SKIP_SCAN | PROCESS_SKIP_ORPHAN_LOCK | ENABLE_CACHE_DEVS,
 			handle, _pv_confirm_single);
 
 	dm_list_iterate_items(pd, &pp->arg_confirm)
@@ -5709,6 +5858,23 @@ do_command:
 
 		label_scan_open_excl(pd->dev);
 
+		if (pp->cachedev) {
+			/*
+			 * cache devs do not hold VG metadata, here we write
+			 * the cachedev PV, and later vgextend will update the
+			 * VG metadata on the other PVs.
+			 */
+			if (!(pv = _cachedev_pv_create(cmd, pd->dev))) {
+				log_error("Failed to create cache device \"%s\".", pv_name);
+				dm_list_move(&pp->arg_fail, &pd->list);
+				continue;
+			}
+			pvl->pv = pv;
+			dm_list_add(&pp->pvs, &pvl->list);
+			log_print_unless_silent("Cache device \"%s\" successfully created.", pv_name);
+			continue;
+		}
+
 		log_debug("Creating a new PV on %s.", pv_name);
 
 		if (!(pv = pv_create(cmd, pd->dev, &pp->pva))) {

tools/toollib.h

@@ -188,6 +188,7 @@ int pvcreate_each_device(struct cmd_context *cmd, struct processing_handle *hand
  */
 struct dm_list *create_pv_list(struct dm_pool *mem, struct volume_group *vg, int argc,
 			    char **argv, int allocatable_only);
+struct dm_list *create_cd_list(struct dm_pool *mem, struct volume_group *vg, int argc, char **argv);
 
 struct dm_list *clone_pv_list(struct dm_pool *mem, struct dm_list *pvs);
 

tools/tools.h

@@ -135,6 +135,8 @@ struct arg_value_group_list {
 #define GET_VGNAME_FROM_OPTIONS  0x00001000
 /* The data read from disk by label scan can be used for vg_read. */
 #define CAN_USE_ONE_SCAN	 0x00002000
+/* Command can process cache devices */
+#define ENABLE_CACHE_DEVS        0x00004000
 
 
 void usage(const char *name);
@@ -267,7 +269,11 @@ int lvconvert_merge_cmd(struct cmd_context *cmd, int argc, char **argv);
 int lvconvert_to_vdopool_cmd(struct cmd_context *cmd, int argc, char **argv);
 int lvconvert_to_vdopool_param_cmd(struct cmd_context *cmd, int argc, char **argv);
 
+int lvconvert_cachevol_cmd(struct cmd_context *cmd, int argc, char **argv);
+
 int pvscan_display_cmd(struct cmd_context *cmd, int argc, char **argv);
 int pvscan_cache_cmd(struct cmd_context *cmd, int argc, char **argv);
 
+int lvcreate_cachevol_cmd(struct cmd_context *cmd, int argc, char **argv);
+
 #endif

tools/vgchange.c

@@ -110,6 +110,10 @@ static int _activate_lvs_in_vg(struct cmd_context *cmd, struct volume_group *vg,
 		if (lv_is_vdo_pool(lv))
 			continue;
 
+		/* cachevol activation is done by activation of origin */
+		if (lv_is_cachevol(lv))
+			continue;
+
 		if (lv_activation_skip(lv, activate, arg_is_set(cmd, ignoreactivationskip_ARG)))
 			continue;
 
@@ -222,6 +226,8 @@ int vgchange_activate(struct cmd_context *cmd, struct volume_group *vg,
 
 	       	if ((lv_open = lvs_in_vg_opened(vg))) {
 			dm_list_iterate_items(lvl, &vg->lvs) {
+				if (lv_is_cachevol(lvl->lv))
+					continue;
 				if (lv_is_visible(lvl->lv) &&
 				    !lv_is_vdo_pool(lvl->lv) && // FIXME: API skip flag missing
 				    !lv_check_not_in_use(lvl->lv, 1)) {

tools/vgextend.c

@@ -93,8 +93,13 @@ static int _vgextend_single(struct cmd_context *cmd, const char *vg_name,
 	if (!archive(vg))
 		return_ECMD_FAILED;
 
-	if (!vg_extend_each_pv(vg, pp))
-		goto_out;
+	if (pp->cachedev) {
+		if (!vg_extend_each_cd(vg, pp))
+			goto_out;
+	} else {
+		if (!vg_extend_each_pv(vg, pp))
+			goto_out;
+	}
 
 	if (arg_is_set(cmd, metadataignore_ARG)) {
 		mda_copies = vg_mda_copies(vg);
@@ -128,14 +133,9 @@ int vgextend(struct cmd_context *cmd, int argc, char **argv)
 	struct pvcreate_params *pp = &vp.pp;
 	unsigned restoremissing = arg_is_set(cmd, restoremissing_ARG);
 	const char *vg_name;
+	int cachedev = (cmd->command->command_enum == vgextend_cachedev_CMD);
 	int ret;
 
-	if (!argc) {
-		log_error("Please enter volume group name and "
-			  "physical volume(s)");
-		return EINVALID_CMD_LINE;
-	}
-
 	vg_name = skip_dev_dir(cmd, argv[0], NULL);
 	argc--;
 	argv++;
@@ -154,6 +154,8 @@ int vgextend(struct cmd_context *cmd, int argc, char **argv)
 	/* pvcreate within vgextend cannot be forced. */
 	pp->force = 0;
 
+	pp->cachedev = cachedev;
+
 	/* Check for old md signatures at the end of devices. */
 	cmd->use_full_md_check = 1;