cache: repair and writeback for cachevol

For a dm-cache LV with an attached cachevol using writeback,
'lvconvert --repair LV' will:

. detach the cachevol
. run cache_repair from the cachevol to a temp LV (or file)
. run cache_writeback to copy blocks from the cachevol
  back to the original LV, using the repaired metadata

tools/args.h

@@ -273,6 +273,9 @@ arg(errorwhenfull_ARG, '\0', "errorwhenfull", bool_VAL, 0, 0,
     "(Also see dm-thin-pool kernel module option no_space_timeout.)\n"
     "See \\fBlvmthin\\fP(7) for more information.\n")
 
+arg(file_long_ARG, '\0', "file", string_VAL, 0, 0,
+    "File name.\n")
+
 arg(force_long_ARG, '\0', "force", 0, ARG_COUNTABLE, 0,
     "Force metadata restore even with thin pool LVs.\n"
     "Use with extreme caution. Most changes to thin metadata\n"

tools/command-lines.in

@@ -746,12 +746,12 @@ FLAGS: SECONDARY_SYNTAX
 # and the LV type is known.
 
 lvconvert --repair LV_cache_cachepool_mirror_raid_thinpool
-OO: --usepolicies, --interval Number, --poolmetadataspare Bool, OO_LVCONVERT
+OO: --usepolicies, --interval Number, --poolmetadataspare Bool, --file_long String, OO_LVCONVERT
 OP: PV ...
 ID: lvconvert_repair
 DESC: Replace failed PVs in a raid or mirror LV.
 DESC: Repair a thin pool.
-DESC: Repair a cache pool.
+DESC: Repair a cache pool or cache.
 RULE: all not lv_is_locked lv_is_pvmove
 RULE: --poolmetadataspare and LV_cache LV_cachepool LV_thinpool
 

tools/lvchange.c

@@ -717,14 +717,6 @@ static int _lvchange_cache(struct cmd_context *cmd,
 	if (!get_cache_params(cmd, &chunk_size, &format, &mode, &name, &settings))
 		goto_out;
 
-	if (seg_is_cache(seg) && lv_is_cache_vol(seg->pool_lv) && (mode == CACHE_MODE_WRITEBACK)) {
-		log_warn("WARNING: repairing a damaged cachevol is not yet possible.");
-		log_warn("WARNING: cache mode writethrough is suggested for safe operation.");
-		if (!arg_count(cmd, yes_ARG) &&
-			yes_no_prompt("Continue using writeback without repair?") == 'n')
-			goto_out;
-	}
-
 	if ((mode != CACHE_MODE_UNSELECTED) &&
 	    (mode != setting_seg->cache_mode) &&
 	    lv_is_cache(lv)) {

tools/lvconvert.c

@@ -2539,10 +2539,409 @@ deactivate_pmslv:
 	return 1;
 }
 
-/* TODO: lots of similar code with  thinpool repair
- *       investigate possible better code sharing...
+/*
+ * Use cache_repair to read metadata from cachevol_lv and
+ * write repaired metadata to new file.
  */
-static int _lvconvert_cache_repair(struct cmd_context *cmd,
+
+static int _run_cache_repair_from_cachevol(struct cmd_context *cmd,
+			     struct logical_volume *cachevol_lv,
+			     const char *meta_file)
+{
+	char cachevol_lv_path[PATH_MAX];
+	char *cachevol_lv_name;
+	const char *dmdir = dm_dir();
+	const char *cache_repair = find_config_tree_str_allow_empty(cmd, global_cache_repair_executable_CFG, NULL);
+	const struct dm_config_node *cn;
+	const struct dm_config_value *cv;
+	const char *argv[MAX_PDATA_ARGS + 7]; /* Max supported args */
+	int args = 0;
+	int status;
+
+	if (!cache_repair || !cache_repair[0]) {
+		log_error("No cache_repair command set in lvm.conf. Repair is disabled.");
+		return 0;
+	}
+
+	if (!(cachevol_lv_name = dm_build_dm_name(cmd->mem, cachevol_lv->vg->name, cachevol_lv->name, NULL))) {
+		log_error("Failed to create dm name for %s.", display_lvname(cachevol_lv));
+		return 0;
+	}
+
+	if (dm_snprintf(cachevol_lv_path, sizeof(cachevol_lv_path), "%s/%s", dmdir, cachevol_lv_name) < 0) {
+		log_error("Failed to create path for %s.", display_lvname(cachevol_lv));
+		return 0;
+	}
+
+	if (!(cn = find_config_tree_array(cmd, global_cache_repair_options_CFG, NULL))) {
+		log_error(INTERNAL_ERROR "Unable to find configuration for global/cache_repair_options");
+		return 0;
+	}
+
+	for (cv = cn->v; cv && args < MAX_PDATA_ARGS; cv = cv->next) {
+		if (cv->type != DM_CFG_STRING) {
+			log_error("Invalid string in config file: "
+				  "global/cache_repair_options");
+			return 0;
+		}
+		argv[++args] = cv->v.str;
+	}
+
+	if (args >= MAX_PDATA_ARGS) {
+		log_error("Too many cache_repair_options set in lvm.conf.");
+		return 0;
+	}
+
+	argv[0] = cache_repair;
+	argv[++args] = "-i";
+	argv[++args] = cachevol_lv_path;
+	argv[++args] = "-o";
+	argv[++args] = meta_file;
+	argv[++args] = NULL;
+
+	if (!exec_cmd(cmd, (const char * const *)argv, &status, 1)) {
+		log_error("The cache_repair command failed (status %d).", status);
+		return 0;
+	}
+
+	log_print("cache_repair of metadata completed from %s to %s", display_lvname(cachevol_lv), meta_file);
+
+	return 1;
+}
+
+static int _run_cache_writeback_from_cachevol(struct cmd_context *cmd,
+			        struct logical_volume *cache_lv,
+			        struct logical_volume *cachevol_lv,
+			        const char *meta_file,
+				uint64_t data_offset_bytes)
+{
+	char orig_path[PATH_MAX];
+	char data_path[PATH_MAX];
+	char *orig_name;
+	char *data_name;
+	const char *dmdir = dm_dir();
+	const char *cache_writeback = "cache_writeback";
+	const char *argv[MAX_PDATA_ARGS + 7]; /* Max supported args */
+	char offset_str[16];
+	int args = 0;
+	int status;
+
+	if (!(orig_name = dm_build_dm_name(cmd->mem, cache_lv->vg->name, cache_lv->name, NULL))) {
+		log_error("Failed to create dm name for %s.", display_lvname(cache_lv));
+	}
+
+	if (!(data_name = dm_build_dm_name(cmd->mem, cachevol_lv->vg->name, cachevol_lv->name, NULL))) {
+		log_error("Failed to create dm name for %s.", display_lvname(cachevol_lv));
+	}
+
+	if (dm_snprintf(orig_path, sizeof(orig_path), "%s/%s", dmdir, orig_name) < 0) {
+		log_error("Failed to create path for %s.", display_lvname(cache_lv));
+		return 0;
+	}
+
+	if (dm_snprintf(data_path, sizeof(data_path), "%s/%s", dmdir, data_name) < 0) {
+		log_error("Failed to create path for %s.", display_lvname(cachevol_lv));
+		return 0;
+	}
+
+	if (dm_snprintf(offset_str, sizeof(offset_str), "%llu", (unsigned long long)data_offset_bytes) < 0) {
+		log_error("Failed to create offset str.");
+		return 0;
+	}
+
+	argv[0] = cache_writeback;
+	argv[++args] = "--metadata-device";
+	argv[++args] = meta_file;
+	argv[++args] = "--origin-device";
+	argv[++args] = orig_path;
+	argv[++args] = "--fast-device";
+	argv[++args] = data_path;
+	argv[++args] = "--fast-device-offset";
+	argv[++args] = offset_str;
+	argv[++args] = "--no-metadata-update";
+	argv[++args] = NULL;
+
+	if (!exec_cmd(cmd, (const char * const *)argv, &status, 1)) {
+		log_error("The cache_writeback command failed (status %d).", status);
+		return 0;
+	}
+
+	log_print("cache_writeback of data completed from %s to %s",
+		  display_lvname(cachevol_lv), display_lvname(cache_lv));
+
+	return 1;
+}
+
+static int _lvconvert_cache_repair_cachevol(struct cmd_context *cmd,
+				   struct logical_volume *cache_lv,
+				   struct dm_list *pvh)
+{
+	char tmp_lv_path[PATH_MAX];
+	char tmp_lv_name[PATH_MAX];
+	const char *tmp_file = NULL;
+	const char *tmp_path = NULL; /* either tmp_file or tmp_lv_path */
+	char zero_sector_buf[512];
+	struct volume_group *vg = cache_lv->vg;
+	struct logical_volume *cachevol_lv;
+	struct logical_volume *tmp_lv;
+	struct lv_segment *cache_seg, *cachevol_seg;
+	uint64_t metadata_len_sectors, data_offset_bytes;
+	uint64_t orig_cachevol_seglen;
+	FILE *fp;
+	uint32_t prealloc_sectors_written = 0;
+	int cache_mode;
+	int ret = 0;
+
+	memset(zero_sector_buf, 0, 512);
+
+	if (!(cache_seg = first_seg(cache_lv)))
+		return_0;
+
+	if (!(cachevol_lv = cache_seg->pool_lv))
+		return_0;
+
+	if (!lv_is_cache_vol(cachevol_lv))
+		return_0;
+
+	if (cache_seg->metadata_start) {
+		/* metadata is always at 0, but check in case it changes in future */
+		log_error("Cannot handle non-zero metadata start locations.");
+		return 0;
+	}
+
+	if (lv_is_active(cache_lv)) {
+		log_error("LV %s must not be active.", display_lvname(cache_lv));
+		return 0;
+	}
+
+	if (lv_is_active(cachevol_lv)) {
+		log_error("LV %s must not be active.", display_lvname(cachevol_lv));
+		return 0;
+	}
+
+	if (vg_is_shared(vg)) {
+		/* cache LV using the cachevol cannot be active elsewhere */
+		if (!lockd_lv(cmd, cache_lv, "ex", 0))
+			return_0;
+
+		/* cachevol cannot be active elsewhere. */
+		if (!lockd_lv(cmd, cachevol_lv, "ex", 0))
+			return_0;
+	}
+
+	cache_mode = cache_seg->cache_mode;
+
+	/*
+	 * For writethrough, the "repair" is to simply remove the
+	 * cachevol without writing back.
+	 */
+	if (cache_mode == CACHE_MODE_WRITETHROUGH) {
+		if (!_lvconvert_split_and_keep_cachevol(cmd, cache_lv, cachevol_lv)) {
+			log_error("Failed to detach writethrough cachevol.");
+			return_0;
+		}
+		log_print("No repair or writeback required for writethrough cache mode.");
+		return 1;
+	}
+
+	metadata_len_sectors = cache_seg->metadata_len;
+
+	/*
+	 * Create a temp file or temp LV to hold a repaired copy of
+	 * cache metadata which is produced by cache_repair and
+	 * consumed by cache_writeback, after which it is deleted.
+	 * User can specify a --file <path> as the temp location,
+	 * otherwise lvconvert creates a temp LV.
+	 */
+	if (arg_is_set(cmd, file_long_ARG)) {
+		tmp_file = arg_str_value(cmd, file_long_ARG, NULL);
+
+		if (!tmp_file || (tmp_file[0] != '/')) {
+			log_error("Full path required to temp file.");
+			tmp_file = NULL;
+			goto out;
+		}
+
+		/* cache_repair wants the destination file preallocated */
+		/* FIXME: do write odirect because that's what cache tools use */
+
+		log_print("Creating %u MB temp file for repair %s",
+			  (uint32_t)(metadata_len_sectors * 512 / 1048576), tmp_file);
+
+		if (!(fp = fopen(tmp_file, "w+"))) {
+			log_error("Failed to open file for metadata %s", tmp_file);
+			tmp_file = NULL;
+			goto out;
+		}
+		while (prealloc_sectors_written < metadata_len_sectors) {
+			if (!fwrite(zero_sector_buf, 512, 1, fp)) {
+				fclose(fp);
+				log_error("Failed to allocate space for metadata in %s", tmp_file);
+				goto out;
+			}
+			prealloc_sectors_written++;
+		}
+		fflush(fp);
+		fclose(fp);
+
+		tmp_path = tmp_file;
+	} else {
+		char *tmp_dm_name;
+		struct lvcreate_params lp = {
+			.activate = CHANGE_AY,
+			.alloc = ALLOC_INHERIT,
+			.major = -1,
+			.minor = -1,
+			.permission = LVM_READ | LVM_WRITE,
+			.pvh = pvh,
+			.read_ahead = DM_READ_AHEAD_NONE,
+			.stripes = 1,
+			.vg_name = vg->name,
+			.zero = 0,
+			.wipe_signatures = 0,
+			.suppress_zero_warn = 1,
+		};
+		lp.lv_name = tmp_lv_name;
+		lp.extents = metadata_len_sectors / vg->extent_size;
+		lp.segtype = get_segtype_from_string(cmd, SEG_TYPE_NAME_STRIPED);
+		dm_list_init(&lp.tags);
+
+		if (dm_snprintf(tmp_lv_name, sizeof(tmp_lv_name), "%s_tmprepair", cache_lv->name) < 0) {
+			log_error("Failed to create temp LV name.");
+			goto out;
+		}
+
+		if (find_lv(vg, tmp_lv_name)) {
+			log_error("Existing temp file for repair exists and must be removed.");
+			goto out;
+		}
+
+		log_print("Creating %u MB temp LV for repair %s/%s",
+			  (uint32_t)(metadata_len_sectors * 512 / 1048576), vg->name, tmp_lv_name);
+
+		if (!(tmp_lv = lv_create_single(vg, &lp))) {
+			log_error("Failed to create temp LV for repair.");
+			goto out;
+		}
+
+		if (!(tmp_dm_name = dm_build_dm_name(cmd->mem, vg->name, tmp_lv->name, NULL))) {
+			log_error("Failed to create dm name for %s.", display_lvname(tmp_lv));
+			goto out;
+		}
+
+		if (dm_snprintf(tmp_lv_path, sizeof(tmp_lv_path), "%s/%s", dm_dir(), tmp_dm_name) < 0) {
+			log_error("Failed to create path for %s.", display_lvname(tmp_lv));
+			goto out;
+		}
+		
+		tmp_path = tmp_lv_path;
+	}
+
+	/* Disable syncing the cache during split (hacky) */
+	cache_seg->cache_mode = CACHE_MODE_WRITETHROUGH;
+
+	if (!_lvconvert_split_and_keep_cachevol(cmd, cache_lv, cachevol_lv)) {
+		log_error("Failed to detach cachevol, not repairing.");
+		return_0;
+	}
+
+	/*
+	 * Note: following the spilt, cache_lv is no longer technically a cache
+	 * and cachevol_lv is no longer technically a cachevol.  The variable
+	 * names represent their previous types.
+	 */
+
+	/*
+	 * Prevent the active LVs from being exposed and used outside of lvm.
+	 */
+	cachevol_lv->status |= LV_TEMPORARY;
+	cache_lv->status |= LV_TEMPORARY;
+
+	/*
+	 * Temporarily change the cachevol LV size to be only the
+	 * metadata size, so cache_repair will only look at the
+	 * metadata portion.
+	 * seg->len and seg->metadata_len units are sectors
+	 */
+	cachevol_seg = first_seg(cachevol_lv);
+	orig_cachevol_seglen = cachevol_seg->len;
+	cachevol_seg->len = metadata_len_sectors / vg->extent_size;
+
+	if (!activate_lv(cmd, cachevol_lv)) {
+		log_error("Failed to activate LV %s for cache repair and writeback.", display_lvname(cachevol_lv));
+		goto out;
+	}
+
+	if (!sync_local_dev_names(cmd)) {
+		log_error("Failed to sync local devices before cache_repair.");
+		goto out;
+	}
+
+	if (!_run_cache_repair_from_cachevol(cmd, cachevol_lv, tmp_path)) {
+		log_error("cache_repair failed, cache data not written back to original volume.");
+		goto out;
+	}
+
+	/*
+	 * Deactivate cachevol LV, and reactivate with correct size.
+	 * Activate cache LV.
+	 */
+
+	if (!deactivate_lv(cmd, cachevol_lv)) {
+		log_error("Failed to deactivate LV %s", cachevol_lv->name);
+		goto out;
+	}
+
+	cachevol_seg->len = orig_cachevol_seglen;
+
+	if (!activate_lv(cmd, cachevol_lv)) {
+		log_error("Failed to activate LV %s for cache repair and writeback.", display_lvname(cachevol_lv));
+		goto out;
+	}
+
+	if (!activate_lv(cmd, cache_lv)) {
+		log_error("Failed to activate LV %s for cache repair and writeback.", display_lvname(cache_lv));
+		goto out;
+	}
+
+	if (!sync_local_dev_names(cmd)) {
+		log_error("Failed to sync local devices before cache_writeback.");
+		goto out;
+	}
+
+	data_offset_bytes = metadata_len_sectors * 512;
+
+	if (!_run_cache_writeback_from_cachevol(cmd, cache_lv, cachevol_lv, tmp_path, data_offset_bytes)) {
+		log_error("cache_writeback failed");
+		goto_out;
+	}
+
+	log_print("cachevol writeback complete, previous cachevol %s is now unused.", cachevol_lv->name);
+	ret = 1;
+out:
+	if (!deactivate_lv(cmd, cache_lv))
+		log_error("Failed to deactivate LV %s", cache_lv->name);
+
+	if (!deactivate_lv(cmd, cachevol_lv))
+		log_error("Failed to deactivate LV %s", cachevol_lv->name);
+
+	if (tmp_file) {
+		if (!unlink(tmp_file)) {
+		}
+	} else if (tmp_lv) {
+		if (!deactivate_lv(cmd, tmp_lv))
+			log_error("Failed to deactivate temp LV %s.", display_lvname(tmp_lv));
+		if (!lv_remove_single(cmd, tmp_lv, DONT_PROMPT, 1))
+			log_error("Failed to remove temp LV %s.", display_lvname(tmp_lv));
+	}
+
+	if (!ret)
+		log_error("Cache data on %s not written back to original LV %s.",
+			  display_lvname(cachevol_lv), display_lvname(cache_lv));
+	return ret;
+}
+
+static int _lvconvert_cache_repair_cachepool(struct cmd_context *cmd,
 				   struct logical_volume *cache_lv,
 				   struct dm_list *pvh, int poolmetadataspare)
 {
@@ -2561,11 +2960,6 @@ static int _lvconvert_cache_repair(struct cmd_context *cmd,
 	struct logical_volume *pmslv;
 	struct logical_volume *mlv;
 
-	if (lv_is_cache(cache_lv) && lv_is_cache_vol(first_seg(cache_lv)->pool_lv)) {
-		log_error("Manual repair required.");
-		return 0;
-	}
-
 	pool_lv = lv_is_cache_pool(cache_lv) ? cache_lv : first_seg(cache_lv)->pool_lv;
 	mlv = first_seg(pool_lv)->metadata_lv;
 
@@ -3603,14 +3997,6 @@ static int _cache_vol_attach(struct cmd_context *cmd,
 	if (!cache_vol_set_params(cmd, cache_lv, lv_fast, poolmetadatasize, chunk_size, cache_metadata_format, cache_mode, policy_name, policy_settings))
 		goto_out;
 
-	if (cache_mode == CACHE_MODE_WRITEBACK) {
-		log_warn("WARNING: repairing a damaged cachevol is not yet possible.");
-		log_warn("WARNING: cache mode writethrough is suggested for safe operation.");
-		if (!arg_count(cmd, yes_ARG) &&
-		    yes_no_prompt("Continue using writeback without repair?") == 'n')
-			goto_out;
-	}
-
 	/*
 	 * vg_write(), suspend_lv(), vg_commit(), resume_lv(),
 	 * where the old LV is suspended and the new LV is resumed.
@@ -3854,7 +4240,7 @@ static int _lvconvert_repair_pvs(struct cmd_context *cmd, struct logical_volume
 	return ret ? ECMD_PROCESSED : ECMD_FAILED;
 }
 
-static int _lvconvert_repair_cachepool_thinpool(struct cmd_context *cmd, struct logical_volume *lv,
+static int _lvconvert_repair_cache_or_thin(struct cmd_context *cmd, struct logical_volume *lv,
 			struct processing_handle *handle)
 {
 	int poolmetadataspare = arg_int_value(cmd, poolmetadataspare_ARG, DEFAULT_POOL_METADATA_SPARE);
@@ -3875,8 +4261,13 @@ static int _lvconvert_repair_cachepool_thinpool(struct cmd_context *cmd, struct
 		if (!_lvconvert_thin_pool_repair(cmd, lv, use_pvh, poolmetadataspare))
 			return_ECMD_FAILED;
 	} else /* cache */ {
-		if (!_lvconvert_cache_repair(cmd, lv, use_pvh, poolmetadataspare))
+		if (lv_is_cache(lv) && lv_is_cache_vol(first_seg(lv)->pool_lv)) {
+			if (!_lvconvert_cache_repair_cachevol(cmd, lv, use_pvh))
 				return_ECMD_FAILED;
+		} else {
+			if (!_lvconvert_cache_repair_cachepool(cmd, lv, use_pvh, poolmetadataspare))
+				return_ECMD_FAILED;
+		}
 	}
 
 	return ECMD_PROCESSED;
@@ -3888,7 +4279,7 @@ static int _lvconvert_repair_single(struct cmd_context *cmd, struct logical_volu
 	if (lv_is_thin_pool(lv) ||
 	    lv_is_cache(lv) ||
 	    lv_is_cache_pool(lv))
-		return _lvconvert_repair_cachepool_thinpool(cmd, lv, handle);
+		return _lvconvert_repair_cache_or_thin(cmd, lv, handle);
 
 	if (lv_is_raid(lv) || lv_is_mirror(lv))
 		return _lvconvert_repair_pvs(cmd, lv, handle);
@@ -3899,12 +4290,6 @@ static int _lvconvert_repair_single(struct cmd_context *cmd, struct logical_volu
 	return ECMD_FAILED;
 }
 
-/*
- * FIXME: add option --repair-pvs to call _lvconvert_repair_pvs() directly,
- * and option --repair-thinpool to call _lvconvert_repair_thinpool().
- * and option --repair-cache to call _lvconvert_repair_cache().
- * and option --repair-cachepool to call _lvconvert_repair_cachepool().
- */
 int lvconvert_repair_cmd(struct cmd_context *cmd, int argc, char **argv)
 {
 	struct processing_handle *handle;