0.6.3-alt1

- massive squashfs tuning

.gear/mkimage-profiles.spec

@@ -1,5 +1,5 @@
 Name: mkimage-profiles
-Version: 0.6.2
+Version: 0.6.3
 Release: alt1
 
 Summary: ALT Linux based distribution metaprofile
@@ -61,6 +61,9 @@ cp -a * %buildroot%mpdir
 %doc README QUICKSTART
 
 %changelog
+* Mon Apr 09 2012 Michael Shigorin <mike@altlinux.org> 0.6.3-alt1
+- massive squashfs tuning
+
 * Mon Apr 02 2012 Michael Shigorin <mike@altlinux.org> 0.6.2-alt1
 - better live-webkiosk and initial live-flightgear
 - cleanup, syslinux, xorg feature tweaks

Makefile

@@ -12,10 +12,12 @@ endif
 endif
 export ARCHES
 
+export PATH := $(CURDIR)/bin:$(PATH)
+
 # supervise target tracing; leave stderr alone
 ifdef REPORT
 export REPORT_PATH := $(shell mktemp --tmpdir mkimage-profiles.report.XXXXXXX)
-POSTPROC := | bin/report-filter > $(REPORT_PATH)
+POSTPROC := | report-filter > $(REPORT_PATH)
 endif
 
 # recursive make considered useful for m-p

conf.d/README

@@ -32,3 +32,14 @@
 - $$(VAR) раскрываются позже, при включении $(CONFIG) и востребовании
   значений; в этом случае их значения могут изменяться до окончания
   конфигурации, а также зависеть от значений других переменных
+
+По спискам пакетов:
+- на этапе экспериментирования можно забивать прямо в описание образа
+- при фиксации состояния стоит воспользоваться существующими списками,
+  а дополнительные оформить как можно более чётко обособленными по тем
+  задачам, для решения которых они и подобраны
+- повторяющиеся логически связанные группы списков может иметь смысл
+  выделить в фичу (см., например, power или x11)
+- если явной фичи не наблюдается, но у группы дистрибутивов намечается
+  заметная общая часть -- её можно выделить в промежуточную цель вида
+  distro/.name, не являющуюся самостоятельно собираемой

conf.d/live.mk

@@ -30,6 +30,9 @@ distro/live-builder: distro/.live-base use/dev/mkimage use/dev/repo
 distro/live-install: distro/.live-base use/syslinux/localboot.cfg
 	@$(call add,LIVE_PACKAGES,live-install)
 
+distro/livecd-install: distro/.live-base use/syslinux/localboot.cfg
+	@$(call add,LIVE_PACKAGES,livecd-install)
+
 distro/live-icewm: distro/.live-desktop use/live/autologin +icewm; @:
 distro/live-razorqt: distro/.live-desktop +razorqt; @:
 distro/live-tde: distro/.live-desktop use/live/ru +tde; @:
@@ -47,7 +50,7 @@ distro/live-webkiosk-mini: distro/.live-kiosk use/live/hooks use/live/ru
 distro/live-webkiosk: distro/live-webkiosk-mini use/live/desktop; @:
 
 distro/live-flightgear: distro/live-icewm use/live/sound use/x11/3d-proprietary
-	@$(call add,LIVE_PACKAGES,FlightGear fgo)
+	@$(call add,LIVE_PACKAGES,FlightGear fgo input-utils)
 	@$(call try,HOMEPAGE,http://www.4p8.com/eric.brasseur/flight_simulator_tutorial.html)
 
 endif

doc/variables.txt

@@ -14,9 +14,9 @@
   + см. ../lib/build.mk
 
 - ARCHES
-  + задаёт набор целевых архитектур образов
+  + задаёт набор целевых архитектур при параметрическом задании APTCONF
   + значение: пусто (по умолчанию авто) либо список через пробел
-  + см. ../Makefile
+  + см. ../Makefile, profiles.mk.sample
 
 - BELL
   + подаёт сигнал после завершения сборки
@@ -81,6 +81,14 @@
   + значение: пусто (по умолчанию) либо любая строка
   + см. ../image.in/Makefile
 
+- SQUASHFS
+  + определяет характер сжатия squashfs для stage2
+  + значение:
+    - пусто (по умолчанию) либо normal: xz
+    - tight: xz с -Xbcj по платформе (лучше, но дольше -- подбор в два прохода)
+    - fast: gzip/lzo (быстрее запаковывается и распаковывается, меньше степень)
+  + см. ../features.in/stage2/stage1/scripts.d/03-test-kernel
+
 пример
 ~~~~~~
 make DEBUG=1 CLEAN=1 distro/syslinux.iso

features.in/live/live/image-scripts.d/40-autologin

@@ -1,7 +1,7 @@
 #!/bin/sh -ef
 
 if [ ! -x /usr/bin/X ]; then
-	echo "warning: autologin feature requested but X not installed" >&2
+	echo "SKIP autologin: X not installed" >&2
 	exit 0
 fi
 

features.in/power/config.mk

@@ -20,4 +20,4 @@ use/power/acpi/powersave: use/power/acpi
 
 # legacy power management
 use/power/apm: use/power
-	@$(call add,COMMON_PACKAGES,apmd)
+	@$(call add,COMMON_PACKAGES,apmd lphdisk)

features.in/stage2/stage1/scripts.d/03-test-kernel

@@ -1,32 +1,55 @@
 #!/bin/sh
+# NB: -f prohibited in this shebang
 # check relevant kernel features availability
 
-GZ_OPTS="-comp gzip"
-XZ_OPTS="-comp xz -b 524288 -noI"
+CONFIGS=/boot/config-*
+GZ_OPTS="-comp gzip -noI"
+XZ_OPTS="-comp xz"
+LZO_OPTS="-comp lzo -noI"
+
+verbose() { [ -n "$GLOBAL_VERBOSE" ] && echo "** $@" >&2; }
+fatal() { echo "** error: $@" >&2; exit 1; }
+
+configured()
+{
+	local option="CONFIG_$1"
+	if grep -q "^$option" $CONFIGS; then
+		verbose "${option%=*} available"
+		return 0
+	else
+		return 1
+	fi
+}
+
+ls $CONFIGS >&/dev/null || fatal "no stage1 kernel config found in /boot"
 
 # test for installer-required filesystems support
-for opt in CONFIG_SQUASHFS CONFIG_AUFS_FS; do
-	if grep -q "^$opt=[my]$" /boot/config-*; then
-		[ -n "$GLOBAL_VERBOSE" ] && echo "** $opt available"
-	else
-		if [ "$?" = 1 ]; then	# a file exists but lacks the pattern
-			echo "** error: stage1 kernel must have $opt support" >&2
-			exit 1
-		fi
-	fi
+for opt in SQUASHFS AUFS_FS; do
+	configured "$opt=[my]" || fatal "stage1 kernel must have $opt support"
 done
 
-# squashfs options: not really neccessary but better than none
-# NB: this config file should be carried over into install2
-if grep -q '^CONFIG_SQUASHFS_XZ=y$' /boot/config-*; then
-	# TODO: figure out if it's generally worth it even on x86:
-	#       if binaries account for less than ~70% of blocks,
-	#       the decompression filter overhead might hurt
-	# NB: there are arm, powerpc and some other filters too
-	if grep -q "^CONFIG_X86" /boot/config-*; then
-		XZ_OPTS="$XZ_OPTS -Xbcj x86"
+verbose "GLOBAL_SQUASHFS: $GLOBAL_SQUASHFS"
+
+options()
+{
+	# squashfs options: not really neccessary but better than none
+	# NB: this config file should be carried over into install2
+	if [ "$GLOBAL_SQUASHFS" != "fast" ] && configured "SQUASHFS_XZ=y"; then
+		# NB: the decompression filter overhead might hurt
+		# NB: there are arm, powerpc and some other filters too
+		if [ "$GLOBAL_SQUASHFS" = "tight" ] && configured "X86"; then
+			XZ_OPTS="$XZ_OPTS -b 524288 -Xbcj x86"
+		else
+			XZ_OPTS="$XZ_OPTS -b 262144 -noI"
+		fi
+		echo "$XZ_OPTS"
+	else	# go fast, or even faster if possible
+		if configured "SQUASHFS_LZO=y"; then
+			echo "$LZO_OPTS"
+		else	# safe default
+			echo "$GZ_OPTS"
+		fi
 	fi
-	echo "PACK_SQUASHFS_OPTS=$XZ_OPTS" > /.image/squashcfg.mk
-else
-	echo "PACK_SQUASHFS_OPTS=$GZ_OPTS" > /.image/squashcfg.mk
-fi
+}
+
+echo "PACK_SQUASHFS_OPTS=-no-recovery `options`" > /.image/squashcfg.mk

lib/functions.mk

@@ -47,7 +47,8 @@ add_feature = $(call add,FEATURES,$(word 2,$(subst /, ,$@)))
 
 # convert tag list into a list of relative package list paths
 # NB: tags can do boolean expressions: (tag1 && !(tag2 || tag3))
-tags = $(and $(strip $(1)),$(addprefix tagged/,$(shell echo "$(1)" | bin/tags2lists pkg.in/lists/tagged)))
+tags = $(and $(strip $(1)),$(addprefix tagged/,$(shell echo "$(1)" \
+       | tags2lists pkg.in/lists/tagged)))
 
 # toplevel Makefile convenience
 addsuffices = $(foreach s,$(1),$(call addsuffix,$s,$(2)))

lib/help.mk

@@ -2,11 +2,11 @@
 
 help/distro:
 	@echo '** available distribution targets:'; \
-	bin/columnize $(sort $(DISTROS:distro/%=%))
+	columnize $(sort $(DISTROS:distro/%=%))
 
 help/ve:
 	@echo '** available virtual environment targets:'; \
-	bin/columnize $(sort $(VES))
+	columnize $(sort $(VES))
 
 help: | help/distro help/space help/ve; @:
 help/space:; @echo

lib/log.mk

@@ -6,6 +6,14 @@ ifndef MKIMAGE_PROFILES
 $(error this makefile is designed to be included in toplevel one)
 endif
 
+# 1.3.22 fixes http://bugzilla.altlinux.org/26217
+HSH_VER_OPTIMAL = 1.3.22
+HSH_VERSION := $(shell hsh -V | sed -n 's/^.* version \([0-9.]\+\).*$$/\1/p')
+
+ifeq (-,$(shell rpmvercmp $(HSH_VERSION) $(HSH_VER_OPTIMAL) | tr -d [0-9]))
+$(info warning: hasher-$(HSH_VERSION) is suboptimal, consider upgrading)
+endif
+
 BUILDLOG ?= $(BUILDDIR)/build.log
 
 # LOG holds a redirecting postprocessor

lib/profile.mk

@@ -15,7 +15,7 @@ if [ -s "$(SYMLINK)" -a "$(NUM_TARGETS)" = 1 ] && \
 then \
 	echo "$(BUILDLINK)"; \
 else \
-	bin/mktmpdir $(BUILDDIR_PREFIX) || exit 200; \
+	mktmpdir $(BUILDDIR_PREFIX) || exit 200; \
 fi; )
 endif
 
@@ -26,9 +26,7 @@ endif
 # even smart caching only hurts when every build goes from scratch
 NO_CACHE ?= 1
 
-PATH := $(CURDIR)/bin:$(PATH)
-
-export BUILDDIR NO_CACHE PATH
+export BUILDDIR NO_CACHE
 
 CONFIG := $(BUILDDIR)/distcfg.mk
 RC := $(HOME)/.mkimage/profiles.mk

main.mk

@@ -6,7 +6,7 @@
 # --- in BUILDDIR
 # 4. build subprofiles and subsequently an image
 
-MKIMAGE_PROFILES = $(dir $(lastword $(MAKEFILE_LIST)))
+MKIMAGE_PROFILES := $(dir $(lastword $(MAKEFILE_LIST)))
 
 # deal with one target at a time
 IMAGE_TARGET := $(firstword $(MAKECMDGOALS))#	ve/generic.tar.gz

pkg.in/lists/README

@@ -8,3 +8,9 @@
 
 Подкаталог tagged/ содержит тегированные списки, имена которых
 удобно получать функцией tags() (см. ../../lib/functions.mk).
+
+Для выявления дубликатов в составе списков служит запускаемый
+с указанием перечня анализируемых файлов скрипт ../../bin/pkgdups;
+пытаться избежать дублей на 100% скорее бесполезно, но бродячие
+устойчивые группы пакетов могут заслуживать выделения отдельным
+списком.

reports.mk

@@ -3,12 +3,17 @@
 all: reports/targets
 
 reports/targets:
-	@if [ -n "$$REPORT_PATH" -a -s "$$REPORT_PATH" ]; then \
-		BUILDDIR="`sed -n 's/^.* BUILDDIR = \(.*\)/\1/p' \
-			"$$REPORT_PATH"`"; \
+	@if ! [ -n "$$REPORT_PATH" -a -s "$$REPORT_PATH" ]; then \
+		exit 0; \
+	fi; \
+	BUILDDIR="`sed -n 's/^.* BUILDDIR = \(.*\)/\1/p' "$$REPORT_PATH"`"; \
+	if type -t dot >&/dev/null; then \
 		REPORT_IMAGE="$$BUILDDIR/targets.png"; \
-		bin/report-targets < "$$REPORT_PATH" \
+		report-targets < "$$REPORT_PATH" \
 		| dot -Tpng -o "$$REPORT_IMAGE" \
-		&& echo "** target graph report: $$REPORT_IMAGE" \
-		&& mv "$$REPORT_PATH" "$$BUILDDIR/targets.log"; \
-	fi
+		&& echo "** target graph report: $$REPORT_IMAGE"; \
+	else \
+		REPORT_DOT="$$BUILDDIR/targets.dot"; \
+		report-targets < "$$REPORT_PATH" > "$$REPORT_DOT" \
+		&& echo "** graphviz missing, target graph dot file: $$REPORT_DOT"; \
+	fi && mv "$$REPORT_PATH" "$$BUILDDIR/targets.log"

sub.in/stage1/Makefile

@@ -38,6 +38,9 @@ endif
 CHROOT_PACKAGES_REGEXP += $(STAGE1_PACKAGES_REGEXP)
 CHROOT_PACKAGES = $(STAGE1_PACKAGES) $(SYSTEM_PACKAGES)
 
+# pass for use/stage2
+GLOBAL_SQUASHFS := $(SQUASHFS)
+
 # scripts prepare bootloader configuration, too
 # NB: we pass tested squashfs options for ../stage2/Makefile to include
 all: | debug prepare-workdir copy-tree run-scripts $(BUILD_PROPAGATOR) \

sub.in/stage2/Makefile

@@ -31,17 +31,29 @@ $(error STAGE1_KFLAVOUR is utterly empty; cannot guess either)
 endif
 
 # need kernel modules only (which require corresponding kernel-image);
-# these go into work chroot; NB: no vmlinuz there
+# these go into work chroot; NB: no vmlinuz there (unless live-install)
 IMAGE_PACKAGES_REGEXP = $(call kpackages, \
 			$(STAGE1_KMODULES) $(STAGE2_KMODULES), \
 			$(STAGE1_KFLAVOUR))
 
+# see also scripts.d/99-elf-stats
+ifdef GLOBAL_SQUASHFS_SORT
+ifeq (tight,$(SQUASHFS))
+CHROOT_PACKAGES += file
+pack-image: PACK_SQUASHFS_OPTS += -sort /.our/elf.lst
+pack-image: CLEANUP_OUTDIR=
+endif
+endif
+
+GLOBAL_SQUASHFS := $(SQUASHFS)
+
 include $(MKIMAGE_PREFIX)/targets.mk
 
 # NB: it's a collector variable, add()ing to a GLOBAL_ results in lost hair
 GLOBAL_CLEANUP_PACKAGES := $(CLEANUP_PACKAGES)
 
-all: | $(GLOBAL_DEBUG) build-image run-image-scripts pack-image \
+all: |  $(GLOBAL_DEBUG) \
+	build-image run-image-scripts run-scripts pack-image \
 	$(GLOBAL_CLEAN_WORKDIR)
 
 # dummy

sub.in/stage2/image-scripts.d/99cleanup-boot

@@ -3,4 +3,4 @@
 # (unless it's a live-install):
 # it's been booted already
 
-[ -x /usr/sbin/live-install ] || rm -f /boot/*
+[ -x /usr/sbin/live-install ] || [ -x /usr/sbin/livecd-install ] || rm -f /boot/*

sub.in/stage2/scripts.d/99-elf-stats

@@ -0,0 +1,48 @@
+#!/bin/sh -e
+# tally up sizes of chroot's ELF binaries for squashfs tuning
+
+# NB: it's unclear so far whether mksquashfs -sort is beneficial
+if [ -z "$GLOBAL_SQUASHFS_SORT" -o "$GLOBAL_SQUASHFS" != "tight" ]; then
+	echo "SKIP elf-stats: looks like it's not needed" >&2
+	exit 0
+fi
+
+exit_handler()
+{
+	local rc=$1
+	rm -f -- "$TMPFILE"
+	exit $rc
+}
+
+# it's controlled environment
+TMPFILE="$(mktemp)"
+
+trap 'exit_handler $?' EXIT HUP PIPE INT QUIT TERM
+
+# a list of ELF binaries (both executables and shared libraries)
+ELFLIST=/.our/elf.lst
+
+cd "$WORKDIR"
+
+# let's parallelize a bit, chroot should be on tmpfs or just-cached
+du -bsx | cut -f1 > "$TMPFILE" &
+
+elf="$(find {,usr/}{lib*,{,s}bin} \
+		-path lib/modules -prune -o \
+		-path usr/lib/locale -prune -o \
+		-type f \
+	| file -nN -e ascii -e compress -e tar -e elf -f - \
+	| sed -n "/: ELF /s/: .*$//p" \
+	| tee "$ELFLIST.in" \
+	| tr '\n' '\0' \
+	| du -cb --files0-from=- \
+	| tail -1 \
+	| cut -f1)"
+
+wait
+read root < "$TMPFILE"
+
+echo "** live chroot ELF ratio: $((100*$elf/$root))% ($elf/$root)" >&2
+
+# add priorities
+sed 's,^.*$,& 1,' < "$ELFLIST.in" > "$ELFLIST"