main.mk introduced, lib/*.mk partially rewritten

The former toplevel Makefile is now toplevel main.mk;
this change allows for multi-target, multi-arch processing
in the current toplevel Makefile.

As the "build" symlink semantics change quite considerably
when one is doing bulk builds (several pruned builddirs might
be useful for comparison), BUILDDIR is now much more likely
to be recreated: the cases when it will persist are when it's
either a single-image build or when the prefix hasn't changed.

There are some more or less subtle bugfixes and enhancements
all over the map as well.

Done within 20111230..20120102 timeframe, actually...

Makefile

@@ -1,75 +1,35 @@
-# steps to build a distribution image:
+# umbrella mkimage-profiles makefile:
-# --- here
+# iterate over multiple goals/arches
-# 1. initialize new profile (BUILDDIR) as a copy of image.in/
-# 2. configure distro
-# 3. copy the needed bits from metaprofile to a new profile
-# --- in BUILDDIR
-# 4. build subprofiles and subsequently an image
 
-help:
+# immediate assignment
-	@echo '** available distribution targets:'
+ifndef ARCHES
-	@echo $(DISTROS) | fmt -sw"$$((COLUMNS>>1))" | column -t
+ifdef ARCH
-	@echo
+ARCHES := $(ARCH)
-	@echo '** available virtual environment targets:'
+else
-	@echo $(VES) | fmt -sw"$$((COLUMNS>>1))" | column -t
+ARCHES := $(shell arch | sed 's/i686/i586/')
-
-MKIMAGE_PROFILES = $(dir $(lastword $(MAKEFILE_LIST)))
-
-# only process the first target (inter-target cleanup is tricky)
-IMAGE_TARGET := $(firstword $(MAKECMDGOALS))#	ve/generic.tar.gz
-ifeq (./,$(dir $(IMAGE_TARGET)))#		convenience fallback
-IMAGE_TARGET := distro/$(IMAGE_TARGET)#		for omitted "distro/"
 endif
-IMAGE_CONF    := $(firstword $(subst ., ,$(IMAGE_TARGET)))# ve/generic
-IMAGE_CLASS   := $(firstword $(subst /, ,$(IMAGE_TARGET)))# ve
-IMAGE_FILE    := $(lastword  $(subst /, ,$(IMAGE_TARGET)))# generic.tar.gz
-IMAGE_NAME    := $(firstword $(subst ., ,$(IMAGE_FILE)))#   generic
-IMAGE_TYPE    := $(subst $(IMAGE_NAME).,,$(IMAGE_FILE))#    tar.gz
-
-# preferences
--include $(HOME)/.mkimage/profiles.mk
-
-# most of the actual work done elsewhere
-include lib/*.mk
-include conf.d/*.mk
-include features.in/*/config.mk
-
-DISTRO_TARGETS := $(shell sed -n 's,^\(distro/[^:.]\+\):.*$$,\1,p' \
-		lib/distro.mk $(wildcard conf.d/*.mk) | sort -u)
-VE_TARGETS := $(shell sed -n 's,^\(ve/[^:.]\+\):.*$$,\1,p' \
-		lib/ve.mk $(wildcard conf.d/*.mk) | sort -u)
-DISTROS := $(call addsuffices,$(DISTRO_EXTS),$(DISTRO_TARGETS)) 
-VES     := $(call addsuffices,$(VE_EXTS),$(VE_TARGETS))
-IMAGES  := $(DISTROS) $(VES)
-
-.PHONY: $(IMAGES) $(DISTRO_TARGETS) $(VE_TARGETS)
-
-### suboptimal but at least clear, reliable and convenient
-all:
-	@n=1; sum=$(words $(DISTROS)); \
-	for distro in $(DISTROS); do \
-		echo "** building $$distro:"; \
-		$(MAKE) --no-print-directory \
-			ALL=$$n/$$sum \
-			BUILDDIR=$(BUILDDIR) \
-			$$distro; \
-		echo; \
-		n=$$(($$n+1)); \
-	done
-
-$(IMAGES): debug \
-	config/with/$(IMAGE_CONF) \
-	config/like/$(IMAGE_CLASS) \
-	config/name/$(IMAGE_NAME) \
-	config/pack/$(IMAGE_TYPE) \
-	build; @:
-
-# convenience shortcut
-$(DISTROS:distro/%=%): %: distro/%
-
-debug:
-ifeq (2,$(DEBUG))
-	@$(foreach v,\
-		$(filter IMAGE_%,$(sort $(.VARIABLES))),\
-		$(warning $v = $($v)))
 endif
+export ARCHES
+
+# recursive make considered useful for m-p
+MAKE += --no-print-directory
+
+.PHONY: clean distclean help
+clean distclean help:
+	@$(MAKE) -f main.mk $@
+
+export NUM_TARGETS := $(words $(MAKECMDGOALS))
+
+%:
+	@n=1; \
+	if [ "$(NUM_TARGETS)" -gt 1 ]; then \
+		n="`echo $(MAKECMDGOALS) \
+		| tr '[[:space:]]' '\n' \
+		| grep -nx "$@" \
+		| cut -d: -f1`"; \
+		echo "** goal: $@ [$$n/$(NUM_TARGETS)]"; \
+	fi; \
+	for ARCH in $(ARCHES); do \
+		$(MAKE) -f main.mk ARCH=$$ARCH $@; \
+	done; \
+	if [ "$$n" -lt "$(NUM_TARGETS)" ]; then echo; fi

image.in/Makefile

@@ -60,5 +60,6 @@ postprocess: | $(addprefix postprocess-,$(sort $(POSTPROCESS_TARGETS)))
 
 debug:
 	@echo "TOPDIR=$(TOPDIR)"
+	@echo "IMAGEDIR=$(IMAGEDIR)"
 	@echo "ARCH=$(ARCH)"
 	@echo "GLOBAL_HSH_APT_CONFIG=$(GLOBAL_HSH_APT_CONFIG)"

image.in/functions.mk

@@ -1,7 +1,7 @@
 # globals
 PKGDIR := $(GLOBAL_BUILDDIR)/pkg
 
-### duplicated from toplevel build.mk, log.mk for sake of "local" builds
+# duplicated from metaprofile makefiles for the sake of "local" builds
 ARCH ?= $(shell arch | sed 's/i686/i586/')
 DATE ?= $(shell date +%Y%m%d)
 

lib/build.mk

@@ -6,8 +6,6 @@ ifndef MKIMAGE_PROFILES
 $(error this makefile is designed to be included in toplevel one)
 endif
 
-export ARCH ?= $(shell arch | sed 's/i686/i586/')
-
 # try not to bog down the system, both CPU and I/O wise
 ifdef NICE
 START := nice $(shell ionice -c3 echo "ionice -c3" 2>/dev/null)
@@ -22,13 +20,18 @@ START += time -f "%E %PCPU %Mk"
 # /usr/bin/{i586,x86_64} are setarch(8) symlinks
 START += $(ARCH)
 
-# to be passed into distcfg.mk
+# to be passed into distcfg.mk; suggestions are welcome
-IMAGEDIR ?= $(shell [ -d "$$HOME/out" -a -w "$$HOME/out" ] \
+IMAGEDIR ?= $(shell \
-	&& echo "$$HOME/out" \
+	if [ -d "$$HOME/out" -a -w "$$HOME/out" ]; then \
-	|| echo "$(BUILDDIR)/out" )
+		echo "$$HOME/out"; \
+	else \
+		dir="`dirname $(BUILDDIR)`/out"; \
+		mkdir -p "$$dir" && echo "$$dir" || echo "/tmp"; \
+	fi; \
+)
 
-# poehali
+# actual build starter
-build: profile/populate
+build-image: profile/populate
 	@if [ -n "$(CHECK)" ]; then \
 		echo "$(TIME) skipping actual image build (CHECK is set)"; \
 		exit; \
@@ -36,22 +39,21 @@ build: profile/populate
 	echo -n "$(TIME) starting image build"; \
 	if [ -n "$(DEBUG)" ]; then \
 		echo ": tail -f $(BUILDLOG)" $(SHORTEN); \
-	else \
-		if [ -n "$(ALL)" ]; then \
-			echo " [$(ALL)]"; \
 	else \
 		echo " (coffee time)"; \
 	fi; \
-	fi; \
 	if $(START) $(MAKE) -C $(BUILDDIR)/ $(LOG); then \
 		echo "$(TIME) done (`tail -1 $(BUILDLOG) | cut -f1 -d.`)"; \
 		tail -200 "$(BUILDLOG)" $(SHORTEN) \
 		| GREP_COLOR="$(ANSI_OK)" \
 		  grep --color=auto '^\*\* image: .*' ||:; \
 	else \
-		echo "$(TIME) failed, see log: $(BUILDLOG)" $(SHORTEN); \
+		echo -n "$(TIME) failed, see log"; \
 		if [ -z "$(DEBUG)" ]; then \
-			echo "$(TIME) (you might want to re-run with DEBUG=1)"; \
+			echo ": $(BUILDLOG)" $(SHORTEN); \
+			echo "$(TIME) (you might want to rerun with DEBUG=1)"; \
+		else \
+			echo " above"; \
 		fi; \
 		tail -200 "$(BUILDLOG)" \
 		| GREP_COLOR="$(ANSI_FAIL)" \

lib/clean.mk

@@ -43,3 +43,14 @@ distclean: clean
 		fi; \
 	fi
 	@rm -f "$(SYMLINK)"
+
+# builddir existing outside read-only metaprofile is less ephemeral
+# than BUILDDIR is -- usually it's unneeded afterwards so just zap it
+postclean: build-image
+	@if [ "$(NUM_TARGETS)" -gt 1 -a -z "$(DEBUG)" ] || \
+	    [ ! -L "$(SYMLINK)" -a "0$(DEBUG)" -lt 2 ]; then \
+		echo "$(TIME) cleaning up after build"; \
+		$(MAKE) -C "$(BUILDDIR)" distclean \
+			GLOBAL_BUILDDIR="$(BUILDDIR)" $(LOG) ||:; \
+		rm -rf "$(BUILDDIR)"; \
+	fi

lib/log.mk

@@ -8,7 +8,7 @@ endif
 
 BUILDLOG ?= $(BUILDDIR)/build.log
 
-# LOG holds a postprocessor
+# LOG holds a redirecting postprocessor
 ifdef DEBUG
 # 1) makefile target; 2) also passed to script hooks
 GLOBAL_DEBUG := debug
@@ -27,3 +27,12 @@ DATE = $(shell date +%Y%m%d)
 TIME = `date +%H:%M:%S`
 
 export BUILDLOG DATE GLOBAL_DEBUG GLOBAL_VERBOSE LOG MAKE SHELL
+
+# brevity postprocessor; not exported, for toplevel use only
+SHORTEN = $(shell \
+	echo -n "| sed"; \
+	if [ -s "$(SYMLINK)" ]; then \
+		echo -n " -e 's,$(BUILDDIR),$(SYMLINK),'"; \
+	fi; \
+	echo -n " -e 's,$(TMP),\$$TMP,' -e 's,$(HOME),~,'"; \
+)

lib/profile.mk

@@ -4,14 +4,19 @@ endif
 
 SYMLINK = build
 
-# this could have come from environment;
+# this could have come from env; or could be symlinked; or is made anew
-# if not, can be symlinked if r/w, or made anew
+# (the reuse rationale is avoiding extra tmpdir lookups)
 # NB: immediate assignment matters
-# NB: PATH has no effect here
 ifndef BUILDDIR
-BUILDDIR := $(shell [ -s "$(SYMLINK)" ] \
+BUILDLINK := $(realpath $(SYMLINK))
-        && realpath "$(SYMLINK)" \
+BUILDDIR  := $(shell \
-        || bin/mktmpdir mkimage-profiles)
+if [ -s "$(SYMLINK)" -a "$(NUM_TARGETS)" = 1 ] && \
+   [ -n "$(findstring $(BUILDDIR_PREFIX).,$(BUILDLINK))" ]; \
+then \
+	echo "$(BUILDLINK)"; \
+else \
+	bin/mktmpdir $(BUILDDIR_PREFIX) || exit 127; \
+fi; )
 endif
 
 # even smart caching only hurts when every build goes from scratch
@@ -24,15 +29,6 @@ export BUILDDIR NO_CACHE PATH
 CONFIG := $(BUILDDIR)/distcfg.mk
 RC := $(HOME)/.mkimage/profiles.mk
 
-# holds a postprocessor; shell test executes in particular situation
-# NB: not exported, for toplevel use only
-SHORTEN = $(shell \
-		if [ -s "$(SYMLINK)" ]; then \
-			echo "| sed 's,$(BUILDDIR),$(SYMLINK),'"; \
-		else \
-			echo "| sed 's,$(TMP),\$$TMP,'"; \
-		fi;)
-
 # step 1: initialize the off-tree mkimage profile (BUILDDIR)
 profile/init: distclean
 	@if [ "`realpath "$(BUILDDIR)/"`" = / ]; then \
@@ -68,7 +64,7 @@ profile/init: distclean
 	fi
 	@if [ -w . ]; then \
 		rm -f "$(SYMLINK)" && \
-		ln -sf "$(BUILDDIR)" "$(SYMLINK)" && \
+		ln -s "$(BUILDDIR)" "$(SYMLINK)" && \
 		echo "$(SYMLINK)/"; \
 	else \
 		echo "$(BUILDDIR)/" $(SHORTEN); \

lib/sugar.mk

@@ -13,3 +13,6 @@ config/pack/%: use/pack/%
 # just preconfigure
 config/name/%:
 	@$(call set,IMAGE_NAME,$*)
+
+# the final thing will pull the rest in
+build: postclean

main.mk

@@ -0,0 +1,88 @@
+# steps to build an image:
+# --- here
+# 1. initialize new profile (BUILDDIR) as a copy of image.in/
+# 2. configure distro
+# 3. copy the needed bits from metaprofile to a new profile
+# --- in BUILDDIR
+# 4. build subprofiles and subsequently an image
+
+MKIMAGE_PROFILES = $(dir $(lastword $(MAKEFILE_LIST)))
+
+# deal with one target at a time
+IMAGE_TARGET := $(firstword $(MAKECMDGOALS))#	ve/generic.tar.gz
+ifeq (./,$(dir $(IMAGE_TARGET)))#		convenience fallback
+IMAGE_TARGET := distro/$(IMAGE_TARGET)#		for omitted "distro/"
+endif
+IMAGE_CONF    := $(firstword $(subst ., ,$(IMAGE_TARGET)))# ve/generic
+IMAGE_CLASS   := $(firstword $(subst /, ,$(IMAGE_TARGET)))# ve
+IMAGE_FILE    := $(lastword  $(subst /, ,$(IMAGE_TARGET)))# generic.tar.gz
+IMAGE_NAME    := $(firstword $(subst ., ,$(IMAGE_FILE)))#   generic
+IMAGE_TYPE    := $(subst $(IMAGE_NAME).,,$(IMAGE_FILE))#    tar.gz
+
+# readjustable
+ifeq (1,$(NUM_TARGETS))
+BUILDDIR_PREFIX ?= mkimage-profiles.build
+else
+BUILDDIR_PREFIX ?= mkimage-profiles.build/$(IMAGE_CONF).$(ARCH)
+endif
+
+# preferences
+-include $(HOME)/.mkimage/profiles.mk
+
+# most of the actual work done elsewhere
+include lib/*.mk
+include conf.d/*.mk
+include features.in/*/config.mk
+
+DISTRO_TARGETS := $(shell sed -n 's,^\(distro/[^:.]\+\):.*$$,\1,p' \
+		lib/distro.mk $(wildcard conf.d/*.mk) | sort -u)
+VE_TARGETS := $(shell sed -n 's,^\(ve/[^:.]\+\):.*$$,\1,p' \
+		lib/ve.mk $(wildcard conf.d/*.mk) | sort -u)
+DISTROS := $(call addsuffices,$(DISTRO_EXTS),$(DISTRO_TARGETS))
+VES     := $(call addsuffices,$(VE_EXTS),$(VE_TARGETS))
+IMAGES  := $(DISTROS) $(VES)
+
+.PHONY: $(IMAGES) $(DISTRO_TARGETS) $(VE_TARGETS)
+.PHONY: debug everything help space
+
+distro/help:
+	@echo '** available distribution targets:'
+	@echo $(DISTROS) | fmt -sw"$$((COLUMNS>>1))" | column -t
+
+ve/help:
+	@echo '** available virtual environment targets:'
+	@echo $(VES) | fmt -sw"$$((COLUMNS>>1))" | column -t
+
+help: | distro/help space ve/help
+space:; @echo
+
+### duplicate but still needed
+everything:
+	@n=1; sum=$(words $(DISTROS)); \
+	for distro in $(DISTROS); do \
+		echo "** building $$distro [$$n/$$sum]:"; \
+		$(MAKE) --no-print-directory \
+			COUNT=$$n/$$sum \
+			BUILDDIR=$(BUILDDIR) \
+			$$distro; \
+		echo; \
+		n=$$(($$n+1)); \
+	done
+
+# config/with/ve/generic config/like/ve config/name/generic config/pack/tar.gz
+$(IMAGES): debug \
+	config/with/$(IMAGE_CONF) \
+	config/like/$(IMAGE_CLASS) \
+	config/name/$(IMAGE_NAME) \
+	config/pack/$(IMAGE_TYPE) \
+	build; @:
+
+# convenience shortcut
+$(DISTROS:distro/%=%): %: distro/%
+
+debug:
+ifeq (2,$(DEBUG))
+	@$(foreach v,\
+		$(filter IMAGE_%,$(sort $(.VARIABLES))),\
+		$(warning $v = $($v)))
+endif