Merge pull request #16282 from poettering/repart-copy-blocks

repart: add new CopyBlocks= setting

TODO

@@ -206,13 +206,6 @@ Features:
   end), in order to maximize dd'ability. Requires libfdisk work, see
   https://github.com/karelzak/util-linux/issues/907
 
-* systemd-repart: optionally, allow specifying a path to initialize new
-  partitions from, i.e. an fs image file or a source device node. This would
-  then turn systemd-repart into a simple installer: with a few .repart files
-  you could replicate the host system on another device. a full installer would
-  then be: "systemd-repart /dev/sda && bootctl install /dev/sda &&
-  systemd-firstboot --image= …"
-
 * systemd-repart: MBR partition table support. Care needs to be taken regarding
   Type=, so that partition definitions can sanely apply to both the GPT and the
   MBR case. Idea: accept syntax "Type=gpt:home mbr:0x83" for setting the types
@@ -248,15 +241,9 @@ Features:
 * systemd-repart: allow disabling growing of specific partitions, or making
   them (think ESP: we don't ever want to grow it, since we cannot resize vfat)
 
-* systemd-repart: add specifier expansion, add especifier that refers to root
-  device node of current system, /usr device node, and matching verity, so that
-  an installer can be made a "copy" installer of the booted OS
-
 * systemd-repart: make it a static checker during early boot for existence and
   absence of other partitions for trusted boot environments
 
-* systemd-repart: allow config of partition uuid
-
 * userdb: allow username prefix searches in varlink API, allow realname and
   realname substr searches in varlink API
 

man/repart.d.xml

@@ -300,13 +300,36 @@
         <term><varname>PaddingMinBytes=</varname></term>
         <term><varname>PaddingMaxBytes=</varname></term>
 
-        <listitem><para>Specifies minimum and maximum size constrains in bytes for the free space after the
+        <listitem><para>Specifies minimum and maximum size constraints in bytes for the free space after the
         partition (the "padding"). Semantics are similar to <varname>SizeMinBytes=</varname> and
         <varname>SizeMaxBytes=</varname>, except that unlike partition sizes free space can be shrunk and can
         be as small as zero. By default no size constraints on padding are set, so that only
         <varname>PaddingWeight=</varname> determines the size of the padding applied.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><varname>CopyBlocks=</varname></term>
+
+        <listitem><para>Takes a path to a regular file, block device node or directory. If specified and the
+        partition is newly created the data from the specified path is written to the newly created
+        partition, on the block level. If a directory is specified the backing block device of the file
+        system the directory is on is determined and the data read directly from that. This option is useful
+        to efficiently replicate existing file systems on the block level on a new partition, for example to
+        build a simple OS installer or OS image builder.</para>
+
+        <para>The file specified here must have a size that is a multiple of the basic block size 512 and not
+        be empty. If this option is used, the size allocation algorithm is slightly altered: the partition is
+        created as least as big as required to fit the data in, i.e. the data size is an additional minimum
+        size value taken into consideration for the allocation algorithm, similar to and in addition to the
+        <varname>SizeMin=</varname> value configured above.</para>
+
+        <para>This option has no effect if the partition it is declared for already exists, i.e. existing
+        data is never overwritten. Note that the data is copied in before the partition table is updated,
+        i.e. before the partition actually is persistently created. This provides robustness: it is
+        guaranteed that the partition either doesn't exist or exists fully populated; it is not possible that
+        the partition exists but is not or only partially populated.</para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><varname>FactoryReset=</varname></term>
 

src/partition/repart.c

@@ -114,6 +114,10 @@ struct Partition {
         FreeArea *padding_area;
         FreeArea *allocated_to_area;
 
+        char *copy_blocks_path;
+        int copy_blocks_fd;
+        uint64_t copy_blocks_size;
+
         LIST_FIELDS(Partition, partitions);
 };
 
@@ -174,6 +178,8 @@ static Partition *partition_new(void) {
                 .padding_max = UINT64_MAX,
                 .partno = UINT64_MAX,
                 .offset = UINT64_MAX,
+                .copy_blocks_fd = -1,
+                .copy_blocks_size = UINT64_MAX,
         };
 
         return p;
@@ -192,6 +198,9 @@ static Partition* partition_free(Partition *p) {
         if (p->new_partition)
                 fdisk_unref_partition(p->new_partition);
 
+        free(p->copy_blocks_path);
+        safe_close(p->copy_blocks_fd);
+
         return mfree(p);
 }
 
@@ -339,7 +348,11 @@ static uint64_t partition_min_size(const Partition *p) {
         }
 
         sz = p->current_size != UINT64_MAX ? p->current_size : HARD_MIN_SIZE;
-        return MAX(p->size_min == UINT64_MAX ? DEFAULT_MIN_SIZE : p->size_min, sz);
+
+        if (p->copy_blocks_size != UINT64_MAX)
+                sz = MAX(p->copy_blocks_size, sz);
+
+        return MAX(p->size_min != UINT64_MAX ? p->size_min : DEFAULT_MIN_SIZE, sz);
 }
 
 static uint64_t partition_max_size(const Partition *p) {
@@ -997,6 +1010,7 @@ static int partition_read_definition(Partition *p, const char *path) {
                 { "Partition", "PaddingMinBytes", config_parse_size4096, 1,  &p->padding_min      },
                 { "Partition", "PaddingMaxBytes", config_parse_size4096, -1, &p->padding_max      },
                 { "Partition", "FactoryReset",    config_parse_bool,     0,  &p->factory_reset    },
+                { "Partition", "CopyBlocks",      config_parse_path,     0,  &p->copy_blocks_path },
                 {}
         };
         int r;
@@ -2126,6 +2140,48 @@ static int context_wipe_and_discard(Context *context, bool from_scratch) {
         return 0;
 }
 
+static int context_copy_blocks(Context *context) {
+        Partition *p;
+        int fd = -1, r;
+
+        assert(context);
+
+        /* Copy in file systems on the block level */
+
+        LIST_FOREACH(partitions, p, context->partitions) {
+                char buf[FORMAT_BYTES_MAX];
+
+                if (p->copy_blocks_fd < 0)
+                        continue;
+
+                if (p->dropped)
+                        continue;
+
+                if (PARTITION_EXISTS(p)) /* Never copy over existing partitions */
+                        continue;
+
+                assert(p->new_size != UINT64_MAX);
+                assert(p->copy_blocks_size != UINT64_MAX);
+                assert(p->new_size >= p->copy_blocks_size);
+
+                if (fd < 0)
+                        assert_se((fd = fdisk_get_devfd(context->fdisk_context)) >= 0);
+
+                if (lseek(fd, p->offset, SEEK_SET) == (off_t) -1)
+                        return log_error_errno(errno, "Failed to seek to partition offset: %m");
+
+                log_info("Copying in '%s' (%s) on block level into partition %" PRIu64 ".", p->copy_blocks_path, format_bytes(buf, sizeof(buf), p->copy_blocks_size), p->partno);
+
+                r = copy_bytes_full(p->copy_blocks_fd, fd, p->copy_blocks_size, 0, NULL, NULL, NULL, NULL);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to copy in data from '%s': %m", p->copy_blocks_path);
+
+                log_info("Copying in of '%s' on block level completed.", p->copy_blocks_path);
+        }
+
+        return 0;
+}
+
 static int partition_acquire_uuid(Context *context, Partition *p, sd_id128_t *ret) {
         struct {
                 sd_id128_t type_uuid;
@@ -2388,6 +2444,10 @@ static int context_write_partition_table(
         if (r < 0)
                 return r;
 
+        r = context_copy_blocks(context);
+        if (r < 0)
+                return r;
+
         LIST_FOREACH(partitions, p, context->partitions) {
                 if (p->dropped)
                         continue;
@@ -2633,6 +2693,87 @@ static int context_can_factory_reset(Context *context) {
         return false;
 }
 
+static int context_open_copy_block_paths(Context *context) {
+        Partition *p;
+        int r;
+
+        assert(context);
+
+        LIST_FOREACH(partitions, p, context->partitions) {
+                _cleanup_close_ int source_fd = -1;
+                uint64_t size;
+                struct stat st;
+
+                assert(p->copy_blocks_fd < 0);
+                assert(p->copy_blocks_size == UINT64_MAX);
+
+                if (PARTITION_EXISTS(p)) /* Never copy over partitions that already exist! */
+                        continue;
+
+                if (!p->copy_blocks_path)
+                        continue;
+
+                source_fd = open(p->copy_blocks_path, O_RDONLY|O_CLOEXEC|O_NOCTTY);
+                if (source_fd < 0)
+                        return log_error_errno(errno, "Failed to open block copy file '%s': %m", p->copy_blocks_path);
+
+                if (fstat(source_fd, &st) < 0)
+                        return log_error_errno(errno, "Failed to stat block copy file '%s': %m", p->copy_blocks_path);
+
+                if (S_ISDIR(st.st_mode)) {
+                        _cleanup_free_ char *bdev = NULL;
+
+                        /* If the file is a directory, automatically find the backing block device */
+
+                        if (major(st.st_dev) != 0)
+                                r = device_path_make_major_minor(S_IFBLK, st.st_dev, &bdev);
+                        else {
+                                dev_t devt;
+
+                                /* Special support for btrfs */
+
+                                r = btrfs_get_block_device_fd(source_fd, &devt);
+                                if (r < 0)
+                                        return log_error_errno(r, "Unable to determine backing block device of '%s': %m", p->copy_blocks_path);
+
+                                r = device_path_make_major_minor(S_IFBLK, devt, &bdev);
+                        }
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to determine block device path for block device backing '%s': %m", p->copy_blocks_path);
+
+                        safe_close(source_fd);
+
+                        source_fd = open(bdev, O_RDONLY|O_CLOEXEC|O_NOCTTY);
+                        if (source_fd < 0)
+                                return log_error_errno(errno, "Failed to open block device '%s': %m", bdev);
+
+                        if (fstat(source_fd, &st) < 0)
+                                return log_error_errno(errno, "Failed to stat block device '%s': %m", bdev);
+
+                        if (!S_ISBLK(st.st_mode))
+                                return log_error_errno(SYNTHETIC_ERRNO(ENOTBLK), "Block device '%s' is not actually a block device, refusing.", bdev);
+                }
+
+                if (S_ISREG(st.st_mode))
+                        size = st.st_size;
+                else if (S_ISBLK(st.st_mode)) {
+                        if (ioctl(source_fd, BLKGETSIZE64, &size) != 0)
+                                return log_error_errno(errno, "Failed to determine size of block device to copy from: %m");
+                } else
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Specified path to copy blocks from '%s' is not a regular file, block device or directory, refusing: %m", p->copy_blocks_path);
+
+                if (size <= 0)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File to copy bytes from '%s' has zero size, refusing.", p->copy_blocks_path);
+                if (size % 512 != 0)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File to copy bytes from '%s' has size that is not multiple of 512, refusing.", p->copy_blocks_path);
+
+                p->copy_blocks_fd = TAKE_FD(source_fd);
+                p->copy_blocks_size = size;
+        }
+
+        return 0;
+}
+
 static int help(void) {
         _cleanup_free_ char *link = NULL;
         int r;
@@ -3213,6 +3354,11 @@ static int run(int argc, char *argv[]) {
         if (r < 0)
                 return r;
 
+        /* Open all files to copy blocks from now, since we want to take their size into consideration */
+        r = context_open_copy_block_paths(context);
+        if (r < 0)
+                return r;
+
         /* First try to fit new partitions in, dropping by priority until it fits */
         for (;;) {
                 if (context_allocate_partitions(context))

src/partition/test-repart.sh

@@ -113,3 +113,34 @@ $D/zzz3 : start=     1185760, size=      591864, type=4F68BCE3-E8CD-4DB1-96E7-FB
 $D/zzz4 : start=     1777624, size=      131072, type=0657FD6D-A4AB-43C4-84E5-0933C84B4F4F, uuid=2AA78CDB-59C7-4173-AF11-C7453737A5D1, name="swap"
 $D/zzz5 : start=     1908696, size=     2285568, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, uuid=A0A1A2A3-A4A5-A6A7-A8A9-AAABACADAEAF, name="custom_label"
 EOF
+
+dd if=/dev/urandom of=$D/block-copy bs=4096 count=10240
+
+cat >$D/definitions/extra2.conf <<EOF
+[Partition]
+Type=linux-generic
+Label=block-copy
+UUID=2a1d97e1d0a346cca26eadc643926617
+CopyBlocks=$D/block-copy
+EOF
+
+$repart $D/zzz --size=3G --dry-run=no --seed=$SEED --definitions=$D/definitions
+
+sfdisk -d $D/zzz | grep -v -e 'sector-size' -e '^$' >$D/populated4
+
+cmp $D/populated4 - <<EOF
+label: gpt
+label-id: EF7F7EE2-47B3-4251-B1A1-09EA8BF12D5D
+device: $D/zzz
+unit: sectors
+first-lba: 2048
+last-lba: 6291422
+$D/zzz1 : start=        2048, size=      591856, type=933AC7E1-2EB4-4F13-B844-0E14E2AEF915, uuid=A6005774-F558-4330-A8E5-D6D2C01C01D6, name="home-$UNAME"
+$D/zzz2 : start=      593904, size=      591856, type=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709, uuid=CE9C76EB-A8F1-40FF-813C-11DCA6C0A55B, name="root-x86-64"
+$D/zzz3 : start=     1185760, size=      591864, type=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709, uuid=AC60A837-550C-43BD-B5C4-9CB73B884E79, name="root-x86-64-2"
+$D/zzz4 : start=     1777624, size=      131072, type=0657FD6D-A4AB-43C4-84E5-0933C84B4F4F, uuid=2AA78CDB-59C7-4173-AF11-C7453737A5D1, name="swap"
+$D/zzz5 : start=     1908696, size=     2285568, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, uuid=A0A1A2A3-A4A5-A6A7-A8A9-AAABACADAEAF, name="custom_label"
+$D/zzz6 : start=     4194264, size=     2097152, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4, uuid=2A1D97E1-D0A3-46CC-A26E-ADC643926617, name="block-copy"
+EOF
+
+cmp --bytes=41943040 --ignore-initial=0:$((512*4194264)) $D/block-copy $D/zzz