dmstats: use canonical path when reporting errors

When a 'dmstats create --filemap' operation fails (e.g. during
open(2), close(2), or dm_stats_create_regions_from_fd()), use the
canonical version of the path. This avoids cryptic/confusing error
messages when symbolic links exist in the path argument given:

  # findmnt /var/lib/libvirt/images -otarget,source
  TARGET                  SOURCE
  /var/lib/libvirt/images /dev/mapper/vg_hex-lv_images

  # readlink /var/lib/libvirt/images/my.img
  /boot/my.img

  # dmstats create --filemap /var/lib/libvirt/images/my.img
  Cannot map file: not a device-mapper device.
  Could not create regions from file /var/lib/libvirt/images/my.img
  Command failed

Using the canonical path the error is immediately obvious:

  # dmstats create --filemap /var/lib/libvirt/images/my.img
  Cannot map file: not a device-mapper device.
  Could not create regions from file /boot/my.img
  Command failed

WHATS_NEW_DM

@@ -1,5 +1,13 @@
 Version 1.02.131 -
 ================================
+  Automatically group regions with 'create --segments' unless --nogroup.
+  Fix resource leak when deleting the first member of a group.
+  Allow --bounds with 'create --filemap' for dmstats.
+  Enable creation of filemap regions with histograms.
+  Enable histogram aggregation for regions with more than one area.
+  Enable histogram aggregation for groups of regions.
+  Add a --filemap option to 'dmstats create' to allow mapping of files.
+  Add dm_stats_create_regions_from_fd() to map file extents to regions.
 
 Version 1.02.130 - 6th July 2016
 ================================

libdm/.exported_symbols.DM_1_02_131

@@ -0,0 +1 @@
+dm_stats_create_regions_from_fd

libdm/libdevmapper.h

@@ -1285,6 +1285,35 @@ uint64_t dm_stats_get_group_id(const struct dm_stats *dms, uint64_t region_id);
 int dm_stats_get_group_descriptor(const struct dm_stats *dms,
 				  uint64_t group_id, char **buf);
 
+/*
+ * Create regions that correspond to the extents of a file in the
+ * filesystem and optionally place them into a group.
+ *
+ * File descriptor fd must reference a regular file, open for reading,
+ * in a local file system that supports the FIEMAP ioctl and that
+ * returns data describing the physical location of extents.
+ *
+ * The file descriptor can be closed by the caller following the call
+ * to dm_stats_create_regions_from_fd().
+ *
+ * The function returns a pointer to an array of uint64_t containing
+ * the IDs of the newly created regions. The array is terminated by the
+ * value DM_STATS_REGIONS_ALL and should be freed using dm_free() when
+ * no longer required.
+ *
+ * Unless nogroup is non-zero the regions will be placed into a group
+ * and the group alias is set to the value supplied.
+ *
+ * The group_id for the new group is equal to the region_id value in
+ * the first array element.
+ *
+ * File mapped histograms will be supported in a future version.
+ */
+uint64_t *dm_stats_create_regions_from_fd(struct dm_stats *dms, int fd,
+					  int group, int precise,
+					  struct dm_histogram *bounds,
+					  const char *alias);
+
 /*
  * Call this to actually run the ioctl.
  */

libdm/libdm-stats.c

@@ -1,5 +1,8 @@
 /*
- * Copyright (C) 2015 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2016 Red Hat, Inc. All rights reserved.
+ *
+ * _stats_get_extents_for_file() based in part on filefrag_fiemap() from
+ * e2fsprogs/misc/filefrag.c. Copyright 2003 by Theodore Ts'o.
  *
  * This file is part of the device-mapper userspace tools.
  *
@@ -16,6 +19,12 @@
 
 #include "math.h" /* log10() */
 
+#include <sys/ioctl.h>
+#include <sys/vfs.h> /* fstatfs */
+#include <linux/fs.h> /* FS_IOC_FIEMAP */
+#include <linux/fiemap.h> /* fiemap */
+#include <linux/magic.h> /* BTRFS_SUPER_MAGIC */
+
 #define DM_STATS_REGION_NOT_PRESENT UINT64_MAX
 #define DM_STATS_GROUP_NOT_PRESENT DM_STATS_GROUP_NONE
 
@@ -76,6 +85,7 @@ struct dm_stats_region {
 	char *aux_data;
 	uint64_t timescale; /* precise_timestamps is per-region */
 	struct dm_histogram *bounds; /* histogram configuration */
+	struct dm_histogram *histogram; /* aggregate cache */
 	struct dm_stats_counters *counters;
 };
 
@@ -83,6 +93,7 @@ struct dm_stats_group {
 	uint64_t group_id;
 	const char *alias;
 	dm_bitset_t regions;
+	struct dm_histogram *histogram;
 };
 
 struct dm_stats {
@@ -330,6 +341,8 @@ static void _stats_group_destroy(struct dm_stats_group *group)
 	if (!_stats_group_present(group))
 		return;
 
+	group->histogram = NULL;
+
 	if (group->alias) {
 		dm_free((char *) group->alias);
 		group->alias = NULL;
@@ -890,6 +903,9 @@ static int _stats_parse_list_region(struct dm_stats *dms,
 	} else
 		region->bounds = NULL;
 
+	/* clear aggregate cache */
+	region->histogram = NULL;
+
 	region->group_id = DM_STATS_GROUP_NOT_PRESENT;
 
 	if (!(region->program_id = dm_strdup(program_id)))
@@ -3168,21 +3184,132 @@ int dm_stats_get_current_region_precise_timestamps(const struct dm_stats *dms)
  * Histogram access methods.
  */
 
+static void _sum_histogram_bins(const struct dm_stats *dms,
+				struct dm_histogram *dmh_aggr,
+				uint64_t region_id, uint64_t area_id)
+{
+	struct dm_stats_region *region;
+	struct dm_histogram_bin *bins;
+	struct dm_histogram *dmh_cur;
+	uint64_t bin;
+
+	region = &dms->regions[region_id];
+	dmh_cur = region->counters[area_id].histogram;
+	bins = dmh_aggr->bins;
+
+	for (bin = 0; bin < dmh_aggr->nr_bins; bin++)
+		bins[bin].count += dmh_cur->bins[bin].count;
+}
+
+/*
+ * Create an aggregate histogram for a sub-divided region or a group.
+ */
+static struct dm_histogram *_aggregate_histogram(const struct dm_stats *dms,
+						 uint64_t region_id,
+						 uint64_t area_id)
+{
+	struct dm_histogram *dmh_aggr, *dmh_cur, **dmh_cachep;
+	int bin, nr_bins, group = 1;
+	uint64_t group_id;
+	size_t hist_size;
+
+	if (area_id == DM_STATS_WALK_REGION) {
+		/* region aggregation */
+		group = 0;
+		if (!_stats_region_present(&dms->regions[region_id]))
+			return_NULL;
+
+		if (!dms->regions[region_id].bounds)
+			return_NULL;
+
+		if (!dms->regions[region_id].counters)
+			return dms->regions[region_id].bounds;
+
+		if (dms->regions[region_id].histogram)
+			return dms->regions[region_id].histogram;
+
+		dmh_cur = dms->regions[region_id].counters[0].histogram;
+		dmh_cachep = &dms->regions[region_id].histogram;
+		nr_bins = dms->regions[region_id].bounds->nr_bins;
+	} else {
+		/* group aggregation */
+		group_id = region_id;
+		area_id = DM_STATS_WALK_GROUP;
+		if (!_stats_group_id_present(dms, group_id))
+			return_NULL;
+
+		if (!dms->regions[group_id].bounds)
+			return_NULL;
+
+		if (!dms->regions[group_id].counters)
+			return dms->regions[group_id].bounds;
+
+		if (dms->groups[group_id].histogram)
+			return dms->groups[group_id].histogram;
+
+		dmh_cur = dms->regions[group_id].counters[0].histogram;
+		dmh_cachep = &dms->groups[group_id].histogram;
+		nr_bins = dms->regions[group_id].bounds->nr_bins;
+	}
+
+	hist_size = sizeof(*dmh_aggr)
+		     + nr_bins * sizeof(struct dm_histogram_bin);
+
+	if (!(dmh_aggr = dm_pool_zalloc(dms->hist_mem, hist_size))) {
+		log_error("Could not allocate group histogram");
+		return 0;
+	}
+
+	dmh_aggr->nr_bins = dmh_cur->nr_bins;
+	dmh_aggr->dms = dms;
+
+	if (!group)
+		_foreach_region_area(dms, region_id, area_id) {
+			_sum_histogram_bins(dms, dmh_aggr, region_id, area_id);
+		}
+	else {
+		_foreach_group_area(dms, group_id, region_id, area_id) {
+			_sum_histogram_bins(dms, dmh_aggr, region_id, area_id);
+		}
+	}
+
+	for (bin = 0; bin < nr_bins; bin++) {
+		dmh_aggr->sum += dmh_aggr->bins[bin].count;
+		dmh_aggr->bins[bin].upper = dmh_cur->bins[bin].upper;
+	}
+
+	/* cache aggregate histogram for subsequent access */
+	*dmh_cachep = dmh_aggr;
+
+	return dmh_aggr;
+}
+
 struct dm_histogram *dm_stats_get_histogram(const struct dm_stats *dms,
 					    uint64_t region_id,
 					    uint64_t area_id)
 {
-	region_id = (region_id == DM_STATS_REGION_CURRENT)
+	int aggr = 0;
-		     ? dms->cur_region : region_id ;
-	area_id = (area_id == DM_STATS_AREA_CURRENT)
-		     ? dms->cur_area : area_id ;
 
-	/* FIXME return histogram sum? Requires bounds check at group time */
+	if (region_id == DM_STATS_REGION_CURRENT) {
-	if (region_id & DM_STATS_WALK_GROUP) {
+		region_id = dms->cur_region;
-		log_err_once("Group histogram data is not supported");
+		if (region_id & DM_STATS_WALK_GROUP) {
-		return NULL;
+			region_id = dms->cur_group;
+			aggr = 1;
+		}
+	} else if (region_id & DM_STATS_WALK_GROUP) {
+		region_id &= ~DM_STATS_WALK_GROUP;
+		aggr = 1;
 	}
 
+	area_id = (area_id == DM_STATS_AREA_CURRENT)
+		   ? dms->cur_area : area_id ;
+
+	if (area_id == DM_STATS_WALK_REGION)
+		aggr = 1;
+
+	if (aggr)
+		return _aggregate_histogram(dms, region_id, area_id);
+
 	if (region_id & DM_STATS_WALK_REGION)
 		region_id &= ~DM_STATS_WALK_REGION;
 
@@ -3733,6 +3860,39 @@ merge:
 	return overlap;
 }
 
+static void _stats_copy_histogram_bounds(struct dm_histogram *to,
+					 struct dm_histogram *from)
+{
+	uint64_t i;
+
+	to->nr_bins = from->nr_bins;
+
+	for (i = 0; i < to->nr_bins; i++)
+		to->bins[i].upper = from->bins[i].upper;
+}
+
+/*
+ * Compare histogram bounds h1 and h2, and return 1 if they match (i.e.
+ * have the same number of bins and identical bin boundary values), or 0
+ * otherwise.
+ */
+static int _stats_check_histogram_bounds(struct dm_histogram *h1,
+					 struct dm_histogram *h2)
+{
+	uint64_t i;
+
+	if (!h1 || !h2)
+		return 0;
+
+	if (h1->nr_bins != h2->nr_bins)
+		return 0;
+
+	for (i = 0; i < h1->nr_bins; i++)
+		if (h1->bins[i].upper != h2->bins[i].upper)
+			return 0;
+	return 1;
+}
+
 /*
  * Create a new group in stats handle dms from the group description
  * passed in group.
@@ -3740,6 +3900,7 @@ merge:
 int dm_stats_create_group(struct dm_stats *dms, const char *members,
 			  const char *alias, uint64_t *group_id)
 {
+	struct dm_histogram *check = NULL, *bounds;
 	int i, count = 0, precise = 0;
 	dm_bitset_t regions;
 
@@ -3750,6 +3911,11 @@ int dm_stats_create_group(struct dm_stats *dms, const char *members,
 
 	if (!(regions = dm_bitset_parse_list(members, NULL))) {
 		log_error("Could not parse list: '%s'", members);
+		return 0;
+	}
+
+	if (!(check = dm_pool_zalloc(dms->hist_mem, sizeof(*check)))) {
+		log_error("Could not allocate memory for bounds check");
 		goto bad;
 	}
 
@@ -3775,14 +3941,20 @@ int dm_stats_create_group(struct dm_stats *dms, const char *members,
 				  FMTu64, i, dms->regions[i].group_id);
 			goto bad;
 		}
-		if (dms->regions[i].bounds) {
-			log_error("Region ID %d: grouping regions with "
-				  "histograms is not yet supported", i);
-			goto bad;
-		}
 		if (dms->regions[i].timescale == 1)
 			precise++;
 
+		/* check for matching histogram bounds */
+		bounds = dms->regions[i].bounds;
+		if (bounds && !check->nr_bins)
+			_stats_copy_histogram_bounds(check, bounds);
+		else if (bounds) {
+			if (!_stats_check_histogram_bounds(check, bounds)) {
+				log_error("All region histogram bounds "
+					  "must match exactly");
+				goto bad;
+			}
+		}
 		count++;
 	}
 
@@ -3796,8 +3968,11 @@ int dm_stats_create_group(struct dm_stats *dms, const char *members,
 	if (!_stats_create_group(dms, regions, alias, group_id))
 		goto bad;
 
+	dm_pool_free(dms->hist_mem, check);
 	return 1;
+
 bad:
+	dm_pool_free(dms->hist_mem, check);
 	dm_bitset_destroy(regions);
 	return 0;
 }
@@ -3886,6 +4061,298 @@ int dm_stats_get_group_descriptor(const struct dm_stats *dms,
 	return 1;
 }
 
+/*
+ * Group a table of region_ids corresponding to the extents of a file.
+ */
+static int _stats_group_file_regions(struct dm_stats *dms, uint64_t *region_ids,
+				     uint64_t count, const char *alias)
+{
+	dm_bitset_t regions = dm_bitset_create(NULL, dms->nr_regions);
+	uint64_t i, group_id = DM_STATS_GROUP_NOT_PRESENT;
+	char *members = NULL;
+	int buflen;
+
+	if (!regions) {
+		log_error("Cannot map file: failed to allocate group bitmap.");
+		return 0;
+	}
+
+	for (i = 0; i < count; i++)
+		dm_bit_set(regions, region_ids[i]);
+
+	buflen = _stats_group_tag_len(dms, regions);
+	members = dm_malloc(buflen);
+
+	if (!members) {
+		log_error("Cannot map file: failed to allocate group "
+			  "descriptor.");
+		dm_bitset_destroy(regions);
+		return 0;
+	}
+
+	if (!_stats_group_tag_fill(dms, regions, members, buflen))
+		goto bad;
+
+	/*
+	 * overlaps should not be possible: overlapping file extents
+	 * returned by FIEMAP imply a kernel bug or a corrupt fs.
+	 */
+	if (!_stats_group_check_overlap(dms, regions, count))
+		log_info("Creating group with overlapping regions.");
+
+	if (!_stats_create_group(dms, regions, alias, &group_id))
+		goto bad;
+
+	dm_free(members);
+	return 1;
+bad:
+	dm_bitset_destroy(regions);
+	dm_free(members);
+	return 0;
+}
+
+static int _stats_add_extent(struct dm_pool *mem, struct fiemap_extent *fm_ext,
+			     uint64_t id)
+{
+	struct _extent extent;
+
+	/* final address of list is unknown */
+	memset(&extent.list, 0, sizeof(extent.list));
+
+	/* convert bytes to dm (512b) sectors */
+	extent.start = fm_ext->fe_physical >> 9;
+	extent.len = fm_ext->fe_length >> 9;
+
+	extent.id = id;
+
+	if (!dm_pool_grow_object(mem, &extent,
+				 sizeof(extent))) {
+		log_error("Cannot map file: failed to grow extent map.");
+		return 0;
+	}
+	return 1;
+
+}
+
+/*
+ * Read the extents of an open file descriptor into a table of struct _extent.
+ *
+ * Based on e2fsprogs/misc/filefrag.c::filefrag_fiemap().
+ *
+ * Copyright 2003 by Theodore Ts'o.
+ *
+ */
+static struct _extent *_stats_get_extents_for_file(struct dm_pool *mem, int fd,
+						   uint64_t *count)
+{
+	uint64_t buf[2048];
+	struct fiemap *fiemap = (struct fiemap *)buf;
+	struct fiemap_extent *fm_ext = &fiemap->fm_extents[0];
+	struct fiemap_extent fm_last = {0};
+	struct _extent *extents;
+	unsigned long long expected = 0;
+	unsigned long long expected_dense = 0;
+	unsigned long flags = 0;
+	unsigned int i, num = 0;
+	int tot_extents = 0, n = 0;
+	int last = 0;
+	int rc;
+
+	memset(buf, 0, sizeof(buf));
+
+	/* space available per ioctl */
+	*count = (sizeof(buf) - sizeof(*fiemap))
+		  / sizeof(struct fiemap_extent);
+
+	/* grow temporary extent table in the pool */
+	if (!dm_pool_begin_object(mem, sizeof(*extents)))
+		return NULL;
+
+	flags |= FIEMAP_FLAG_SYNC;
+
+	do {
+		/* start of ioctl loop - zero size and set count to bufsize */
+		fiemap->fm_length = ~0ULL;
+		fiemap->fm_flags = flags;
+		fiemap->fm_extent_count = *count;
+
+		/* get count-sized chunk of extents */
+		rc = ioctl(fd, FS_IOC_FIEMAP, (unsigned long) fiemap);
+		if (rc < 0) {
+			rc = -errno;
+			if (rc == -EBADR)
+				log_err_once("FIEMAP failed with unknown "
+					     "flags %x.", fiemap->fm_flags);
+			goto bad;
+		}
+
+		/* If 0 extents are returned, then more ioctls are not needed */
+		if (fiemap->fm_mapped_extents == 0)
+			break;
+
+		for (i = 0; i < fiemap->fm_mapped_extents; i++) {
+			expected_dense = fm_last.fe_physical +
+					 fm_last.fe_length;
+			expected = fm_last.fe_physical +
+				   fm_ext[i].fe_logical - fm_last.fe_logical;
+			if ((fm_ext[i].fe_logical != 0)
+			    && (fm_ext[i].fe_physical != expected)
+			    && (fm_ext[i].fe_physical != expected_dense)) {
+				tot_extents++;
+				if (!_stats_add_extent(mem, fm_ext + i,
+						       tot_extents - 1))
+					goto bad;
+			} else {
+				expected = 0;
+				if (!tot_extents)
+					tot_extents = 1;
+				if (fm_ext[i].fe_logical == 0)
+					if (!_stats_add_extent(mem, fm_ext + i,
+							       tot_extents - 1))
+						goto bad;
+			}
+			num += tot_extents;
+			if (fm_ext[i].fe_flags & FIEMAP_EXTENT_LAST)
+				last = 1;
+			fm_last = fm_ext[i];
+			n++;
+		}
+
+		fiemap->fm_start = (fm_ext[i - 1].fe_logical +
+				    fm_ext[i - 1].fe_length);
+	} while (last == 0);
+
+	if (!tot_extents) {
+		log_error("Cannot map file: no allocated extents.");
+		goto bad;
+	}
+
+	/* return total number of extents */
+	*count = tot_extents;
+	return dm_pool_end_object(mem);
+bad:
+	dm_pool_abandon_object(mem);
+	return NULL;
+}
+
+/*
+ * Create a set of regions representing the extents of a file and
+ * return a table of uint64_t region_id values. The number of regions
+ * created is returned in the memory pointed to by count (which must be
+ * non-NULL).
+ */
+static uint64_t *_stats_create_file_regions(struct dm_stats *dms, int fd,
+					    struct dm_histogram *bounds,
+					    int precise, uint64_t *count)
+{
+	struct _extent *extents = NULL;
+	uint64_t *regions = NULL, i;
+	char *hist_arg = NULL;
+	struct statfs fsbuf;
+	struct stat buf;
+
+	if (fstatfs(fd, &fsbuf)) {
+		log_error("fstatfs failed for fd %d", fd);
+		return 0;
+	}
+
+	if (fsbuf.f_type == BTRFS_SUPER_MAGIC) {
+		log_error("Cannot map file: btrfs does not provide "
+			  "physical FIEMAP extent data.");
+		return 0;
+	}
+
+	if (fstat(fd, &buf)) {
+		log_error("fstat failed for fd %d", fd);
+		return 0;
+	}
+
+	if (!(buf.st_mode & S_IFREG)) {
+		log_error("Not a regular file");
+		return 0;
+	}
+
+	if (!dm_is_dm_major(major(buf.st_dev))) {
+		log_error("Cannot map file: not a device-mapper device.");
+		return 0;
+	}
+
+	if (!(extents = _stats_get_extents_for_file(dms->mem, fd, count)))
+		return_0;
+
+        if (bounds) {
+                /* _build_histogram_arg enables precise if vals < 1ms. */
+                if (!(hist_arg = _build_histogram_arg(bounds, &precise)))
+                        goto_out;
+        }
+
+	/* make space for end-of-table marker */
+	if (!(regions = dm_malloc((1 + *count) * sizeof(*regions)))) {
+		log_error("Could not allocate memory for region IDs.");
+		goto out;
+	}
+
+	for (i = 0; i < *count; i++) {
+		if (!_stats_create_region(dms, regions + i, extents[i].start,
+					  extents[i].len, -1, precise, hist_arg,
+					  dms->program_id, "")) {
+			log_error("Failed to create region " FMTu64 " of "
+				  FMTu64 " at " FMTu64 ".", i, *count,
+				  extents[i].start);
+			goto out_remove;
+		}
+	}
+	regions[*count] = DM_STATS_REGION_NOT_PRESENT;
+
+	dm_pool_free(dms->mem, extents);
+	return regions;
+
+out_remove:
+	/* clean up regions after create failure */
+	for (--i; i != DM_STATS_REGION_NOT_PRESENT; i--) {
+		if (!dm_stats_delete_region(dms, i))
+			log_error("Could not delete region " FMTu64 ".", i);
+	}
+
+out:
+	dm_pool_free(dms->mem, extents);
+	dm_free(regions);
+	return NULL;
+}
+
+
+uint64_t *dm_stats_create_regions_from_fd(struct dm_stats *dms, int fd,
+					  int group, int precise,
+					  struct dm_histogram *bounds,
+					  const char *alias)
+{
+	uint64_t *regions, count = 0;
+
+	if (alias && !group) {
+		log_error("Cannot set alias without grouping regions.");
+		return NULL;
+	}
+
+	regions = _stats_create_file_regions(dms, fd, bounds, precise, &count);
+	if (!regions)
+		return_0;
+
+	if (!group)
+		return regions;
+
+	/* refresh handle */
+	if (!dm_stats_list(dms, NULL))
+		goto_out;
+
+	if (!_stats_group_file_regions(dms, regions, count, alias))
+		goto_out;
+
+	return regions;
+out:
+	dm_free(regions);
+	return NULL;
+}
+
 /*
  * Backward compatible dm_stats_create_region() implementations.
  *

man/dmstats.8.in

@@ -87,6 +87,9 @@ dmstats \(em device-mapper statistics management
 .  IR area_size ]
 .  RB [ \-\-bounds
 .  IR \%histogram_boundaries ]
+.  RB [ \-\-filemap
+.  IR path ]
+.  RB [ \-\-nogroup ]
 .  RB [ \-\-precise ]
 .  RB [ \-\-start
 .  IR start_sector
@@ -145,8 +148,6 @@ dmstats \(em device-mapper statistics management
 .  RI [ device_name ]
 .  RB [ \-\-histogram ]
 .  OPT_PROGRAMS
-.  RB [ \-\-statstype
-.  IR type_list ]
 .  RB [ \-\-units
 .  IR units ]
 .  OPT_OBJECTS
@@ -190,8 +191,6 @@ dmstats \(em device-mapper statistics management
 .  IR sort_fields ]
 .  RB [ \-S | \-\-select
 .  IR selection ]
-.  RB [ \-\-statstype
-.  IR type_list ]
 .  RB [ \-\-units
 .  IR units ]
 .  RB [ \-\-nosuffix ]
@@ -299,6 +298,14 @@ When peforming a list or report, include objects of type group in the
 results.
 .
 .HP
+.BR \-\-filemap
+.IR path
+.br
+Instead of creating regions specified by command line options, open
+the file found at \fBpath\fP, and create regions corresponding to the
+locations of the on-disk extents allocated to the file.
+.
+.HP
 .BR \-\-groupid
 .IR id
 .br
@@ -356,6 +363,12 @@ Specify the major number.
 Specify the minor number.
 .
 .HP
+.BR \-\-nogroup
+.br
+When creating regions mapping the extents of a file in the file
+system, do not create a group or set an alias.
+.
+.HP
 .BR \-\-nosuffix
 .br
 Suppress the suffix on output sizes.  Use with \fB\-\-units\fP
@@ -443,23 +456,13 @@ optional suffix selects units of:
 .HP
 .BR \-\-segments
 .br
-Create a new statistics region for each target contained in the target
+When used with \fBcreate\fP, create a new statistics region for each
-device. This causes a separate region to be allocated for each segment
+target contained in the given device(s). This causes a separate region
-of the device.
+to be allocated for each segment of the device.
-.
-.HP
-.BR \-\-statstype
-.IR type_list
-.br
-Filter the types of statistics object included in a report or listing
-according to the provided list. A report may include areas, regions,
-and user-defined groups of regions that report aggregate data for all
-group members.
 
-The list may be a single object type, a comma separated list of types,
+The newly created regions are automatically placed into a group unless
-or the special value 'all'.
+the \fB\-\-nogroup\fP option is given. When grouping is enabled a group
-
+alias may be specified using the \fB\-\-alias\fP option.
-The currently available object types are 'area', 'region' and 'group'.
 .
 .HP
 .BR \-\-units
@@ -538,8 +541,27 @@ device-mapper kernel statistics subsystem.
 By default dmstats creates regions with a \fBprogram_id\fP of
 "dmstats".
 
-On success the \fBregion_id\fP of the newly created region is printed to
+On success the \fBregion_id\fP of the newly created region is printed
-stdout.
+to stdout.
+
+If the \fB\-\-filemap\fP option is given with a regular file as the
+\fBpath\fP argument, instead of creating regions with parameters
+specified on the command line, \fBdmstats\fP will open the file located
+at \fBpath\fP and create regions corresponding to the physical extents
+allocated to the file. This can be used to monitor statistics for
+individual files in the file system, for example, virtual machine
+images, swap areas, or large database files.
+
+To work with the \fB\-\-filemap\fP option, files must be located on a
+local file system, backed by a device-mapper device, that supports
+physical extent data using the FIEMAP ioctl (Ext4 and XFS for e.g.).
+
+By default regions that map a file are placed into a group and the
+group alias is set to the basename of the file. This behaviour can be
+overridden with the \fB\-\-alias\fP and \fB\-\-nogroup\fP options.
+
+Use the \fB\-\-group\fP option to only display information for groups
+when listing and reporting.
 .
 .HP
 .CMD_DELETE
@@ -573,6 +595,9 @@ regions is given as a comma-separated list of region identifiers. A
 continuous range of identifers spanning from \fBR1\fP to \fBR2\fP may
 be expressed as '\fBR1\fP-\fBR2\fP'.
 
+Regions that have a histogram configured can be grouped: in this case
+the number of histogram bins and their bounds must match exactly.
+
 On success the group list and newly created \fBgroup_id\fP are
 printed to stdout.
 .
@@ -592,13 +617,13 @@ regardless of region program ID values.
 By default only regions and groups are included in list output. If
 \fB\-v\fP or \fB\-\-verbose\fP is given the report will also include a
 row of information for each configured group and for each area contained
-in each region displayed (regions that contain a single area are by
+in each region displayed.
-default omitted from the verbose list since their properties are
-identical to the area that they contain - to view all regions regardless
-of the number of areas they contain use \fB\-\-statstype\fP).
 
-Specific combinations of objects may be selected using the
+Regions that contain a single area are by default omitted from the
-\fB\-\-statstype\fP option.
+verbose list since their properties are identical to the area that they
+contain - to view all regions regardless of the number of areas present
+use \fB\-\-region\fP). To also view the areas contained within regions
+use \fB\-\-area\fP.
 
 If \fB\-\-histogram\fP is given the report will include the bin count
 and latency boundary values for any configured histograms.
@@ -625,8 +650,9 @@ values and latency boundaries.
 If the \fB\-\-relative\fP is used the default histogram field displays
 bin values as a percentage of the total number of I/Os.
 
-Object types (areas, regions and groups) are selected using the
+Object types (areas, regions and groups) to include in the report are
-\fB\-\-statstype\fP option.
+selected using the \fB\-\-area\fP, \fB\-\-region\fP, and \fB\-\-group\fP
+options.
 .
 .HP
 .CMD_UNGROUP
@@ -1019,6 +1045,14 @@ vg00-lvol1: Created new region with 1 area(s) as region ID 1
 .br
 vg00-lvol1: Created new region with 1 area(s) as region ID 2
 .P
+Create regions mapping the file vm.img and place them into a group with
+the alias set to the same name as the file.
+.br
+#
+.B dmstats create --filemap vm.img
+.br
+vm.img: Created new group with 112 region(s) as group ID 3.
+.P
 Print raw counters for region 4 on device d0
 .br
 #

tools/dmsetup.c

@@ -171,6 +171,7 @@ enum {
 	DEFERRED_ARG,
 	SELECT_ARG,
 	EXEC_ARG,
+	FILEMAP_ARG,
 	FORCE_ARG,
 	GID_ARG,
 	GROUP_ARG,
@@ -187,6 +188,7 @@ enum {
 	MODE_ARG,
 	NAMEPREFIXES_ARG,
 	NOFLUSH_ARG,
+	NOGROUP_ARG,
 	NOHEADINGS_ARG,
 	NOLOCKFS_ARG,
 	NOOPENCOUNT_ARG,
@@ -4611,6 +4613,24 @@ static int _bind_stats_device(struct dm_stats *dms, const char *name)
 	return 1;
 }
 
+static int _bind_stats_from_fd(struct dm_stats *dms, int fd)
+{
+	int major, minor;
+	struct stat buf;
+
+	if (fstat(fd, &buf)) {
+		log_error("fstat failed for fd %d.", fd);
+		return 0;
+	}
+
+	major = (int) MAJOR(buf.st_dev);
+	minor = (int) MINOR(buf.st_dev);
+
+	if (!dm_stats_bind_devno(dms, major, minor))
+		return_0;
+	return 1;
+}
+
 static int _stats_clear_one_region(struct dm_stats *dms, uint64_t region_id)
 {
 
@@ -4748,6 +4768,56 @@ static uint64_t _nr_areas_from_step(uint64_t len, int64_t step)
 	return (len / step) + !!(len % (uint64_t) step);
 }
 
+/* maximum length of a string representation of an integer */
+#define max_int_strlen(i) (strlen(#i))
+#define MAX_UINT64_STRLEN max_int_strlen(UINT64_MAX)
+static int _stats_group_segments(struct dm_stats *dms, uint64_t *region_ids,
+				 int count, const char *alias)
+{
+	/* NULL, commas, and count * region_id */
+	size_t bufsize = 1 + count + count * MAX_UINT64_STRLEN;
+	char *this_region, *regions = NULL;
+	uint64_t group_id;
+	int r, i;
+
+	this_region = regions = dm_malloc(bufsize);
+
+	if (!regions) {
+		log_error("Could not allocate memory for region_id table.");
+		return 0;
+	}
+
+	for (i = 0; i < count; i++) {
+		/*
+		 * We don't expect large numbers of segments (compared to e.g.
+		 * --filemap): use a fixed-size buffer based on the number of
+		 *  region identifiers and do not collapse continuous ranges
+		 *  of identifiers in the group descriptor argument.
+		 */
+		r = dm_snprintf(this_region, bufsize, FMTu64 "%s", region_ids[i],
+				(i < (count - 1)) ? "," : "");
+		if (r < 0)
+			goto_bad;
+		this_region += r;
+		bufsize -= r;
+	}
+
+	/* refresh handle */
+	if (!(r = dm_stats_list(dms, NULL)))
+		goto bad;
+
+	if ((r = dm_stats_create_group(dms, regions, alias, &group_id)))
+		printf("Grouped regions %s as group ID " FMTu64 "%s%s\n",
+		       regions, group_id, (alias) ? " with alias " : "",
+		       (alias) ? : "");
+	else
+		log_error("Failed to create group for regions %s", regions);
+
+bad:
+	dm_free(regions);
+	return r;
+}
+
 /*
  * Create a single region starting at start and spanning len sectors,
  * or, if the segments argument is no-zero create one region for each
@@ -4764,8 +4834,9 @@ static int _do_stats_create_regions(struct dm_stats *dms,
 {
 	uint64_t this_start = 0, this_len = len, region_id = UINT64_C(0);
 	const char *devname = NULL, *histogram = _string_args[BOUNDS_ARG];
-	int r = 0, precise = _switches[PRECISE_ARG];
+	int r = 0, count = 0, precise = _switches[PRECISE_ARG];
 	struct dm_histogram *bounds = NULL; /* histogram bounds */
+	uint64_t *region_ids = NULL; /* segments */
 	char *target_type, *params; /* unused */
 	struct dm_task *dmt;
 	struct dm_info info;
@@ -4774,6 +4845,12 @@ static int _do_stats_create_regions(struct dm_stats *dms,
 	if (histogram && !(bounds = dm_histogram_bounds_from_string(histogram)))
 		return_0;
 
+	if (_switches[ALIAS_ARG] && _switches[NOGROUP_ARG]) {
+		log_error("Cannot set alias with --nogroup.");
+		dm_stats_destroy(dms);
+		return 0;
+	}
+
 	if (!(dmt = dm_task_create(DM_DEVICE_TABLE))) {
 		dm_histogram_bounds_destroy(bounds);
 		dm_stats_destroy(dms);
@@ -4798,6 +4875,11 @@ static int _do_stats_create_regions(struct dm_stats *dms,
 	if (!(devname = dm_task_get_name(dmt)))
 		goto_out;
 
+	if (!segments || (info.target_count == 1))
+		region_ids = &region_id;
+	else
+		region_ids = dm_malloc(info.target_count * sizeof(*region_ids));
+
 	do {
 		uint64_t segment_start, segment_len;
 		next = dm_get_next_target(dmt, next, &segment_start, &segment_len,
@@ -4818,10 +4900,10 @@ static int _do_stats_create_regions(struct dm_stats *dms,
 			 */
 			this_start = (segments) ? segment_start : start;
 			this_len = (segments) ? segment_len : this_len;
-			if (!dm_stats_create_region(dms, &region_id,
+			if (!(r = dm_stats_create_region(dms, &region_ids[count],
-						    this_start, this_len, step,
+							 this_start, this_len, step,
-						    precise, bounds,
+							 precise, bounds,
-						    program_id, user_data)) {
+							 program_id, user_data))) {
 				log_error("%s: Could not create statistics region.",
 					  devname);
 				goto out;
@@ -4829,18 +4911,195 @@ static int _do_stats_create_regions(struct dm_stats *dms,
 
 			printf("%s: Created new region with "FMTu64" area(s) as "
 			       "region ID "FMTu64"\n", devname,
-			       _nr_areas_from_step(this_len, step), region_id);
+			       _nr_areas_from_step(this_len, step),
+			       region_ids[count++]);
 		}
 	} while (next);
-	r = 1;
+
+	if (!_switches[NOGROUP_ARG] && segments)
+		r = _stats_group_segments(dms, region_ids, count,
+					  _string_args[ALIAS_ARG]);
 
 out:
+	if (region_ids != &region_id)
+		dm_free(region_ids);
+
 	dm_task_destroy(dmt);
 	dm_stats_destroy(dms);
 	dm_histogram_bounds_destroy(bounds);
 	return r;
 }
 
+/*
+ * Returns the full absolute path, or NULL if the path could
+ * not be resolved.
+ */
+static char *_get_abspath(const char *path)
+{
+	char *_path;
+
+#ifdef HAVE_CANONICALIZE_FILE_NAME
+	_path = canonicalize_file_name(path);
+#else
+	/* FIXME Provide alternative */
+	log_error(INTERNAL_ERROR "Unimplemented _get_abspath.");
+	_path = NULL;
+#endif
+	return _path;
+}
+
+static int _stats_create_file(CMD_ARGS)
+{
+	const char *alias, *program_id = DM_STATS_PROGRAM_ID;
+	const char *histogram = _string_args[BOUNDS_ARG];
+	uint64_t *regions, *region, count = 0;
+	struct dm_histogram *bounds = NULL;
+	char *path, *abspath = NULL;
+	int group, fd, precise;
+	struct dm_stats *dms;
+
+	if (_switches[AREAS_ARG] || _switches[AREA_SIZE_ARG]) {
+		log_error("--filemap is incompatible with --areas and --area-size.");
+		return 0;
+	}
+
+	if (_switches[START_ARG] || _switches[LENGTH_ARG]) {
+		log_error("--filemap is incompatible with --start and --length.");
+		return 0;
+	}
+
+	if (_switches[SEGMENTS_ARG]) {
+		log_error("--filemap and --segments are incompatible.");
+		return 0;
+	}
+
+	if (_switches[USER_DATA_ARG]) {
+		log_error("--userdata is not yet supported with --filemap.");
+		return 0;
+	}
+
+	/* _stats_create_file does not use _process_all() */
+	if (names) {
+		log_error("Device argument not compatible with --filemap.");
+		return 0;
+	} else {
+		if (argc || _switches[UUID_ARG] || _switches[MAJOR_ARG]) {
+			log_error("--uuid, --major, and device argument are "
+				  "incompatible with --filemap.");
+			return 0;
+		}
+		if (_switches[ALL_DEVICES_ARG]) {
+			log_error("--alldevices is incompatible with "
+				  "--filemap.");
+			return 0;
+		}
+	}
+
+	if (_switches[PRECISE_ARG]) {
+		if (!dm_stats_driver_supports_precise()) {
+			log_error("Using --precise requires driver version "
+				  "4.32.0 or later.");
+			return 0;
+		}
+	}
+
+	if (_switches[BOUNDS_ARG]) {
+		if (!dm_stats_driver_supports_histogram()) {
+			log_error("Using --bounds requires driver version "
+				  "4.32.0 or later.");
+			return 0;
+		}
+	}
+
+	if (histogram && !(bounds = dm_histogram_bounds_from_string(histogram)))
+		return_0;
+
+	if (_switches[PROGRAM_ID_ARG])
+		program_id = _string_args[PROGRAM_ID_ARG];
+	if (!strlen(program_id) && !_switches[FORCE_ARG])
+		program_id = DM_STATS_PROGRAM_ID;
+
+	path = _string_args[FILEMAP_ARG];
+	precise = _int_args[PRECISE_ARG];
+	group = !_switches[NOGROUP_ARG];
+
+	if (!(abspath = _get_abspath(path))) {
+		log_error("Could not canonicalize file name: %s", path);
+		return 0;
+	}
+
+	if (!(dms = dm_stats_create(DM_STATS_PROGRAM_ID)))
+		return_0;
+
+	fd = open(abspath, O_RDONLY);
+
+	if (fd < 0) {
+		log_error("Could not open %s for reading", abspath);
+		goto bad;
+	}
+
+	if (!_bind_stats_from_fd(dms, fd))
+		goto_bad;
+
+	if (!strlen(program_id))
+		/* force creation of a region with no id */
+		dm_stats_set_program_id(dms, 1, NULL);
+
+	if (group && !_switches[ALIAS_ARG])
+		alias = dm_basename(abspath);
+	else if (group)
+		alias = _string_args[ALIAS_ARG];
+	else if (!_switches[ALIAS_ARG])
+		alias = NULL;
+	else {
+		log_error("Cannot set alias with --nogroup.");
+		goto bad;
+	}
+
+	regions = dm_stats_create_regions_from_fd(dms, fd, group, precise,
+						  bounds, alias);
+
+	if (close(fd))
+		log_error("Error closing %s", abspath);
+
+	fd = -1;
+
+	if (!regions) {
+		log_error("Could not create regions from file %s", abspath);
+		goto bad;
+	}
+
+	for (region = regions; *region != DM_STATS_REGIONS_ALL; region++) {
+		count++;
+	}
+
+	if (group) {
+		printf("%s: Created new group with "FMTu64" region(s) as "
+		       "group ID "FMTu64".\n", path, count, regions[0]);
+	} else {
+		region = regions;
+		do
+			printf("%s: Created new region with 1 area as "
+			       "region ID "FMTu64".\n", path, *region);
+		while (*(++region) != DM_STATS_REGIONS_ALL);
+	}
+
+	dm_free(regions);
+	dm_free(abspath);
+	dm_stats_destroy(dms);
+	return 1;
+
+bad:
+	if (abspath)
+		dm_free(abspath);
+
+	if ((fd > -1) && close(fd))
+		log_error("Error closing %s", path);
+
+	dm_stats_destroy(dms);
+	return 0;
+}
+
 static int _stats_create(CMD_ARGS)
 {
 	struct dm_stats *dms;
@@ -4876,6 +5135,10 @@ static int _stats_create(CMD_ARGS)
 			return 0;
 	}
 
+	if (_switches[FILEMAP_ARG])
+		return _stats_create_file(cmd, subcommand, argc, argv,
+					  names, multiple_devices);
+
 	if (names)
 		name = names->name;
 	else {
@@ -5030,6 +5293,7 @@ static int _stats_delete(CMD_ARGS)
 			log_error("Could not delete statistics group.");
 			goto out;
 		}
+		printf("Deleted statistics group " FMTu64 ".\n", group_id);
 	} else if (_switches[ALL_REGIONS_ARG]) {
 		dm_stats_foreach_region(dms) {
 			region_id = dm_stats_get_current_region(dms);
@@ -5682,24 +5946,6 @@ static int _process_tree_options(const char *options)
 	return 1;
 }
 
-/*
- * Returns the full absolute path, or NULL if the path could
- * not be resolved.
- */
-static char *_get_abspath(const char *path)
-{
-	char *_path;
-
-#ifdef HAVE_CANONICALIZE_FILE_NAME
-	_path = canonicalize_file_name(path);
-#else
-	/* FIXME Provide alternative */
-	log_error(INTERNAL_ERROR "Unimplemented _get_abspath.");
-	_path = NULL;
-#endif
-	return _path;
-}
-
 static char *parse_loop_device_name(const char *dev, const char *dev_dir)
 {
 	char *buf;
@@ -5984,6 +6230,7 @@ static int _process_switches(int *argcp, char ***argvp, const char *dev_dir)
 		{"deferred", 0, &ind, DEFERRED_ARG},
 		{"select", 1, &ind, SELECT_ARG},
 		{"exec", 1, &ind, EXEC_ARG},
+		{"filemap", 1, &ind, FILEMAP_ARG},
 		{"force", 0, &ind, FORCE_ARG},
 		{"gid", 1, &ind, GID_ARG},
 		{"group", 0, &ind, GROUP_ARG},
@@ -5998,6 +6245,7 @@ static int _process_switches(int *argcp, char ***argvp, const char *dev_dir)
 		{"minor", 1, &ind, MINOR_ARG},
 		{"mode", 1, &ind, MODE_ARG},
 		{"nameprefixes", 0, &ind, NAMEPREFIXES_ARG},
+		{"nogroup", 0, &ind, NOGROUP_ARG},
 		{"noflush", 0, &ind, NOFLUSH_ARG},
 		{"noheadings", 0, &ind, NOHEADINGS_ARG},
 		{"nolockfs", 0, &ind, NOLOCKFS_ARG},
@@ -6147,6 +6395,10 @@ static int _process_switches(int *argcp, char ***argvp, const char *dev_dir)
 			_switches[CLEAR_ARG]++;
 		if (c == 'c' || c == 'C' || ind == COLS_ARG)
 			_switches[COLS_ARG]++;
+		if (ind == FILEMAP_ARG) {
+			_switches[FILEMAP_ARG]++;
+			_string_args[FILEMAP_ARG] = optarg;
+		}
 		if (c == 'f' || ind == FORCE_ARG)
 			_switches[FORCE_ARG]++;
 		if (c == 'r' || ind == READ_ONLY)
@@ -6306,6 +6558,8 @@ static int _process_switches(int *argcp, char ***argvp, const char *dev_dir)
 			_switches[NAMEPREFIXES_ARG]++;
 		if (ind == NOFLUSH_ARG)
 			_switches[NOFLUSH_ARG]++;
+		if (ind == NOGROUP_ARG)
+			_switches[NOGROUP_ARG]++;
 		if (ind == NOHEADINGS_ARG)
 			_switches[NOHEADINGS_ARG]++;
 		if (ind == NOLOCKFS_ARG)