tar2vm: rewrote as tar2fs

Overview of the changes:
- ARM support: separate ext2 /boot, no LILO
- avoid race condition with devmapper
- trap ERR so that -e in shebang doesn't result in extra cleanup hassle
- configurable root filesystem type (ext4 by default)
- jumps through parted hoops

Details:

1. LILO is x86-specific while the rest of the script can be used
   to prepare e.g. Marvell ArmadaXP or CuBox images; we can generally
   count on uboot supporting ext2 for relatively sane platforms but
   not ext4 that would be a better root filesystem performance-wise.

2. Apparently /dev/mapper/loopXpY can be still missing at the time
   when kpartx returns and pop up a bit later... sit there, wait
   and check for it.

3. If something went wrong with any command of the script it would bail out
   due to -e in shebang; it is now better to clean up the loopback device
   and its mappings in this situation either.

4. One size doesn't fit all, really.

5. The parted sizing was sloppy as in broken, now it's just half insane.
   Someone's decision to stick units and auto-alignment knobs into
   a single one was apparently hilarious...

   http://www.gnu.org/software/parted/manual/parted.html#unit

Manual loop/dm cleanup is described in documentation just in case.

/boot size meter is suboptimal in terms of additional I/O incurred,
will be most likely rewritten to make use of advance "du -s".

bin/tar2fs

@@ -0,0 +1,236 @@
+#!/bin/bash -ex
+# usage:
+# tar2fs chroot.tar image.raw [size_in_bytes [fstype]]
+
+. shell-error
+
+if [ $# -lt 2 ]; then
+	fatal "error: tar2fs needs at least two arguments"
+fi
+
+# this needs env_keep sudo setup to actually work
+if [ -n "$GLOBAL_BUILDDIR" ]; then
+	WORKDIR="$GLOBAL_BUILDDIR/vmroot"
+else
+	WORKDIR="$(mktemp --tmpdir -d vmroot-XXXXX)"
+fi
+
+[ -n "$WORKDIR" ] || fatal "couldn't come up with suitable WORKDIR"
+
+[ -n "$GLOBAL_DEBUG" ] || message "WORKDIR: $WORKDIR"
+
+MB=1048576		# a parted's "megabyte" in bytes is *broken*
+SIZE_FACTOR=2		# multiply the sizes found by this value
+
+CUR_BOUNDARY=0		# align first partition at 1MB for performance (+1)
+
+INITRD_MODULES=
+BOOTFSTYPE=
+BOOTPART=
+
+case "`arch`" in	# NB: sudo => no GLOBAL_ will do either; mind qemu-*
+*86*)
+	# NB: different storage modules might be needed for non-kvm
+	INITRD_MODULES="sd_mod ata_piix"
+	BLOCKDEV="/dev/sda"	# might be /dev/vda for virtio
+	ROOTPART="1"
+	;;
+arm*)
+	BOOTFSTYPE="ext2"	# support expected in every sane target uboot
+	BLOCKDEV="/dev/mmcblk0p"	# ...hopefully...
+	BOOTPART="1"
+	ROOTPART="2"
+	;;
+esac
+
+# figure out the part taken by /boot in the given tarball
+boot_size() {
+	if [ -n "$BOOTPART" ]; then
+		tar tvf "$1" \
+		| awk ' \
+			BEGIN { sum=0 }
+			/^-.* \.\/boot\// { sum=sum+$3 }
+			END { print sum }'
+	else
+		echo "0"
+	fi
+}
+
+# parted wrapper for convenience
+parting() { parted "$LOOPDEV" --align optimal --script -- "$@"; }
+
+# unfortunately parted is insane enough to lump alignment controls
+# into unit controls so creating adjacent partitions sized in MiB
+# is not as straightforward as it might be... thus "+1" padding;
+# see also http://www.gnu.org/software/parted/manual/parted.html#unit
+mkpart() {
+	# a bit different start/end semantics to handle end of device too
+	local start="$(($CUR_BOUNDARY + 1))"	# yes, we lose a megabyte
+	if [ -n "$1" ]; then
+		CUR_BOUNDARY="$(($start + $1))"
+		local end="$CUR_BOUNDARY"MiB
+	else
+		local end="-1s"	# last sector of the image
+	fi
+	parting mkpart primary ext2 "$start"MiB "$end"
+}
+
+# a tarball containing chroot with a kernel
+TAR="$1"
+[ -s "$TAR" ] || fatal "source tarball doesn't really exist"
+
+# a path to the image to be generated
+IMG="$2"
+[ -d "$(dirname "$IMG")" ] || fatal "target directory doesn't exist"
+
+# 0 means auto; if a value is specified it holds (no /boot subtracted)
+ROOTSIZE="$3"
+[ -n "$ROOTSIZE" -a "$ROOTSIZE" != 0 ] || unset ROOTSIZE
+
+# image size in bytes (note the final ceil rounding to megabytes)
+TARSIZE="$(stat -Lc %s "$TAR")"
+# /boot size in that tarball
+BOOTSIZE="$(boot_size "$TAR")"
+DEFSIZE="$(($SIZE_FACTOR * ($TARSIZE - $BOOTSIZE)))"	# (exact sizes)
+ROOTSIZE="$((${ROOTSIZE:-$DEFSIZE} + $MB - 1))"
+# image and /boot sizes in megabytes
+ROOTSIZEM="$(($ROOTSIZE / $MB))"
+BOOTSIZEM="$((($SIZE_FACTOR * $BOOTSIZE + $MB - 1) / $MB))"
+
+# tested to work: ext[234], jfs
+# NB: xfs doesn't have a spare sector for the bootloader
+ROOTFSTYPE="${4:-ext4}"
+
+# single root partition hardwired so far,
+# add another image for home/data/swap if needed
+ROOTDEV="$BLOCKDEV$ROOTPART"
+
+# last preparations...
+MKFS="mkfs.$ROOTFSTYPE ${BOOTFSTYPE:+mkfs.$BOOTFSTYPE}"
+for i in losetup sfdisk parted kpartx $MKFS; do
+	if ! type -t "$i" >&/dev/null; then
+		fatal "$i required but not found in host system"
+	fi
+done
+
+LOOPDEV="$(losetup --find)"	# would be sad about non-binary megabytes too
+ROOTFS="$WORKDIR/chroot"
+
+BOOTFS=
+if [ -n "$BOOTPART" ]; then
+	BOOTFS="$ROOTFS/boot"
+fi
+
+exit_handler() {
+	rc=$?
+	if [ -n "$ROOTFS" ]; then
+		umount ${BOOTFS:+"$BOOTFS"} "$ROOTFS"{/dev,/proc,/sys,}
+
+		if [ -n "$LOOPDEV" ]; then
+			kpartx -d "$LOOPDEV"
+			losetup --detach "$LOOPDEV"
+		fi
+		rm -r -- "$ROOTFS"
+		rmdir -- "$WORKDIR"
+	fi
+	exit $rc
+}
+
+# handle -e in shebang as well
+trap exit_handler EXIT ERR
+
+# prepare disk image and a filesystem inside it
+rm -f -- "$IMG"
+OFFSET="$(($CUR_BOUNDARY + $BOOTSIZEM + $ROOTSIZEM - 1))"
+dd if=/dev/zero of="$IMG" conv=notrunc bs=$MB count=1 seek="$OFFSET"
+losetup "$LOOPDEV" "$IMG"
+
+parting mklabel msdos
+
+if [ -n "$BOOTPART" ]; then
+	BOOTDEV="$BLOCKDEV$BOOTPART"
+	mkpart "$BOOTSIZEM"
+fi
+
+# not ROOTSIZEM but "the rest"; somewhat non-trivial arithmetics lurk in parted
+mkpart
+
+kpartx -a "$LOOPDEV"
+LOOPROOT="/dev/mapper/$(basename "$LOOPDEV")p$ROOTPART"
+
+# might take a tiny bit of time to actually appear
+for i in `seq 10`; do
+	[ -b "$LOOPROOT" ] && break || sleep 0.1
+done
+
+mkfs."$ROOTFSTYPE" "$LOOPROOT"
+
+if [ -n "$BOOTPART" ]; then
+	LOOPBOOT="/dev/mapper/$(basename "$LOOPDEV")p$BOOTPART"
+	mkfs."$BOOTFSTYPE" "$LOOPBOOT"
+fi
+
+# mount and populate it
+mkdir -pm755 "$ROOTFS"
+mount "$LOOPROOT" "$ROOTFS"
+
+if [ -n "$BOOTPART" ]; then
+	mkdir -pm700 "$BOOTFS"
+	mount "$LOOPBOOT" "$BOOTFS"
+fi
+
+tar -C "$ROOTFS" --numeric-owner -xf "$TAR"
+for i in /dev /proc /sys; do mount --bind "$i" "$ROOTFS$i"; done
+
+# loop device so lilo could work...
+echo "$LOOPROOT / $ROOTFSTYPE relatime 1 1" >> "$ROOTFS/etc/fstab"
+
+# target device at once
+if [ -n "$BOOTPART" ]; then
+	echo "$BOOTDEV /boot $BOOTFSTYPE defaults 1 2" >> "$ROOTFS/etc/fstab"
+fi
+
+echo "MODULES_PRELOAD=$INITRD_MODULES $ROOTFSTYPE" >> "$ROOTFS/etc/initrd.mk"
+
+KERNEL="$(readlink $ROOTFS/boot/vmlinuz | sed 's,vmlinuz-,,')"
+chroot "$ROOTFS" make-initrd -k "$KERNEL"
+
+# ...target device too
+sed -i "s,$LOOPROOT,$ROOTDEV," "$ROOTFS/etc/fstab"
+
+if [ -x /sbin/lilo ]; then
+	# configure and install bootloader
+	REGEXP='^Disk .*: ([0-9]+) cylinders, ([0-9]+) heads, ([0-9]+) sectors/track*$'
+	set -- $(sfdisk -l "$LOOPDEV" | grep -E "$REGEXP" | sed -r "s@$REGEXP@\1 \2 \3@")
+
+	LILO_COMMON="lba32
+delay=1
+vga=0
+image=/boot/vmlinuz
+  initrd=/boot/initrd.img
+  append=\"root=$ROOTDEV rootdelay=3 quiet\"
+  label=linux"
+
+	cat > "$ROOTFS"/etc/lilo-loop.conf <<-EOF
+	boot=$LOOPDEV
+	disk=$LOOPDEV
+	  bios=0x80
+	  cylinders=$1
+	  heads=$2
+	  sectors=$3
+	  partition=$LOOPROOT
+	  start=63
+	$LILO_COMMON
+	EOF
+
+	chroot "$ROOTFS" lilo -C /etc/lilo-loop.conf
+
+	cat > "$ROOTFS"/etc/lilo.conf <<-EOF
+	boot=$BLOCKDEV
+	$LILO_COMMON
+	EOF
+fi
+
+if [ -n "$SUDO_USER" ]; then
+	chown "$SUDO_USER" "$IMG" "$ROOTFS" "$WORKDIR"
+fi

bin/tar2vm

@@ -1,134 +0,0 @@
-#!/bin/bash -e
-# usage:
-# tar2vm chroot.tar image.raw [size_in_bytes]
-
-. shell-error
-
-if [ $# -lt 2 ]; then
-	fatal "error: tar2vm needs at least two arguments"
-fi
-
-# this needs env_keep sudo setup to actually work
-if [ -n "$GLOBAL_BUILDDIR" ]; then
-	WORKDIR="$GLOBAL_BUILDDIR/vmroot"
-else
-	WORKDIR="$(mktemp --tmpdir -d vmroot-XXXXX)"
-fi
-
-[ -n "$WORKDIR" ] || fatal "couldn't come up with suitable WORKDIR"
-
-[ -n "$GLOBAL_DEBUG" ] || message "WORKDIR: $WORKDIR"
-
-# a tarball containing chroot with a kernel
-TAR="$1"
-[ -s "$TAR" ] || fatal "source tarball doesn't really exist"
-
-# a path to the image to be generated
-IMG="$2"
-[ -d "$(dirname "$IMG")" ] || fatal "target directory doesn't exist"
-
-# image size in bytes (256M is a fallback)
-TARSIZE="$(stat -Lc %s "$TAR")"
-DEFSIZE="$((2 * $TARSIZE))"
-DISKSIZE="${3:-${DEFSIZE:-268435456}}"
-# ...and in megabytes
-DISKSIZEM="$(($DISKSIZE / 1048576))"
-
-# tested to work: ext[234], jfs
-# NB: xfs doesn't have a spare sector for the bootloader
-ROOTFSTYPE="${4:-ext4}"
-
-# single root partition hardwired so far,
-# add another image for swap if needed
-ROOTDEV="/dev/sda1"
-
-# last preparations...
-for i in losetup sfdisk parted kpartx mkfs."$ROOTFSTYPE"; do
-	if ! type -t "$i" >&/dev/null; then
-		fatal "$i required but not found"
-	fi
-done
-
-LOOPDEV="$(losetup --find)"
-ROOTFS="$WORKDIR/chroot"
-
-exit_handler()
-{
-	rc=$?
-	if [ -n "$ROOTFS" ]; then
-		umount "$ROOTFS"{/dev,/proc,/sys,}
-		if [ -n "$LOOPDEV" ]; then
-			kpartx -d "$LOOPDEV"
-			losetup --detach "$LOOPDEV"
-		fi
-		rm -r -- "$ROOTFS"
-		rmdir -- "$WORKDIR"
-	fi
-	exit $rc
-}
-
-trap exit_handler EXIT
-
-# prepare disk image and a filesystem inside it
-rm -f -- "$IMG"
-dd if=/dev/zero of="$IMG" conv=notrunc \
-	bs=1 count=1 seek="$(($DISKSIZE - 1))"
-losetup "$LOOPDEV" "$IMG"
-
-parted --script "$LOOPDEV" mklabel msdos
-parted --script "$LOOPDEV" mkpart primary ext2 1 "$DISKSIZEM"
-
-kpartx -a "$LOOPDEV"
-LOOPDEV1="/dev/mapper/$(basename "$LOOPDEV")p1"
-
-mkfs."$ROOTFSTYPE" "$LOOPDEV1"
-
-# mount and populate it
-mkdir -pm755 "$ROOTFS"
-mount "$LOOPDEV1" "$ROOTFS"
-tar -C "$ROOTFS" --numeric-owner -xf "$TAR"
-for i in /dev /proc /sys; do mount --bind "$i" "$ROOTFS$i"; done
-
-# NB: different storage modules might be needed for non-kvm
-echo "$LOOPDEV1 / $ROOTFSTYPE defaults 1 1" >> "$ROOTFS/etc/fstab"
-echo "MODULES_PRELOAD=sd_mod ata_piix $ROOTFSTYPE" >> "$ROOTFS/etc/initrd.mk"
-
-KERNEL="$(readlink $ROOTFS/boot/vmlinuz | sed 's,vmlinuz-,,')"
-chroot "$ROOTFS" make-initrd -k "$KERNEL"
-
-sed -i "s,$LOOPDEV1,$ROOTDEV," "$ROOTFS/etc/fstab"
-
-# configure and install bootloader
-REGEXP='^Disk .*: ([0-9]+) cylinders, ([0-9]+) heads, ([0-9]+) sectors/track*$'
-set -- $(sfdisk -l "$LOOPDEV" | grep -E "$REGEXP" | sed -r "s@$REGEXP@\1 \2 \3@")
-
-LILO_COMMON="lba32
-delay=1
-vga=0
-image=/boot/vmlinuz
-  initrd=/boot/initrd.img
-  append=\"root=$ROOTDEV rootdelay=3\"
-  label=linux"
-
-cat > "$ROOTFS"/etc/lilo-loop.conf << EOF
-boot=$LOOPDEV
-disk=$LOOPDEV
-  bios=0x80
-  cylinders=$1
-  heads=$2
-  sectors=$3
-  partition=$LOOPDEV1
-  start=63
-$LILO_COMMON
-EOF
-
-chroot "$ROOTFS" lilo -C /etc/lilo-loop.conf
-
-cat > "$ROOTFS"/etc/lilo.conf << EOF
-boot=${ROOTDEV%[0-9]*}
-$LILO_COMMON
-EOF
-
-if [ -n "$SUDO_USER" ]; then
-	chown "$SUDO_USER" "$IMG" "$ROOTFS" "$WORKDIR"
-fi

doc/params.txt

@@ -109,7 +109,7 @@
 * VM_SIZE
 ** задаёт размер несжатого образа виртуальной машины в байтах
 ** значение: пусто (по умолчанию двойной размер чрута) или целое
-** см. ../features.in/build-vm/lib/90-build-vm.mk, ../bin/tar2vm
+** см. ../features.in/build-vm/lib/90-build-vm.mk, ../bin/tar2fs
 
 [float]
 === пример ===

doc/vm.txt

@@ -2,14 +2,14 @@
 
 *ВНИМАНИЕ:* заключительная операция создания образа жёсткого диска
 из архива с содержимым корневой файловой системы требует доступа
-к sudo и разрешения на выполнение скрипта bin/tar2vm в корневом
+к sudo и разрешения на выполнение скрипта bin/tar2fs в корневом
 каталоге метапрофиля при установке mkimage-profiles из пакета
 (это в планах исправить, но подход к libguestfs пока успехом
 не увенчался).
 
 Соответствующий фрагмент конфигурации sudo(8) может выглядеть как:
 
-  mike ALL=NOPASSWD: /usr/share/mkimage-profiles/bin/tar2vm
+  mike ALL=NOPASSWD: /usr/share/mkimage-profiles/bin/tar2fs
 
 При работе с локальной копией mkimage-profiles.git следует иметь
 в виду, что предоставлять недоверенному пользователю право выполнять
@@ -25,3 +25,10 @@
 Пример сборки и запуска VM:
 
   $ make ROOTPW=reallysecret1 vm/bare.img && kvm -hda ~/out/bare.img
+
+Если при сборке образа файловой системы произойдёт сбой, может оказаться
+нужным вручную освободить используемые loop-устройства, например, так:
+
+  # losetup -a
+  # kpartx -d /dev/loop0
+  # losetup -d /dev/loop0

features.in/build-vm/generate.mk

@@ -1,4 +1,4 @@
-# copy tar2vm helper into generated profile to enable standalone builds
+# copy tar2fs helper into generated profile to enable standalone builds
 
 all:
-	@install -pD $(MKIMAGE_PROFILES)/bin/tar2vm $(BUILDDIR)/bin/tar2vm
+	@install -pD $(MKIMAGE_PROFILES)/bin/tar2fs $(BUILDDIR)/bin/tar2fs

features.in/build-vm/lib/90-build-vm.mk

@@ -10,6 +10,8 @@ IMAGE_PACKAGES = $(DOT_BASE) \
 # intermediate chroot archive
 VM_TARBALL := $(IMAGE_OUTDIR)/$(IMAGE_NAME).tar
 VM_RAWDISK := $(IMAGE_OUTDIR)/$(IMAGE_NAME).raw
+VM_FSTYPE ?= ext4
+VM_SIZE ?= 0
 
 check-sudo:
 	@if ! type -t sudo >&/dev/null; then \
@@ -18,9 +20,9 @@ check-sudo:
 	fi
 
 prepare-image: check-sudo
-	@if ! sudo $(TOPDIR)/bin/tar2vm \
+	@if ! sudo $(TOPDIR)/bin/tar2fs \
-		"$(VM_TARBALL)" "$(VM_RAWDISK)" $$VM_SIZE; then \
+		"$(VM_TARBALL)" "$(VM_RAWDISK)"  $(VM_SIZE) $(VM_FSTYPE); then \
-		echo "** error: sudo tar2vm failed, see also doc/vm.txt" >&2; \
+		echo "** error: sudo tar2fs failed, see also doc/vm.txt" >&2; \
 		exit 1; \
 	fi