From 788cad885ef0fc0e3e1302bdee6aa5761eac5998 Mon Sep 17 00:00:00 2001 From: Michael Shigorin Date: Mon, 19 Mar 2012 13:42:10 +0200 Subject: [PATCH] initial target tracing support and reporting The purpose is being able to examine particular target interdependency graph for a given image having been configured to avoid convoluted dependencies (loops in particular). The implementation is based on SHELL hook hint by John Graham-Cumming: http://cmcrossroads.com/ask-mr-make/6535-tracing-rule-execution-in-gnu-make --- Makefile | 25 +++++++++++++------- bin/report-filter | 4 ++++ bin/report-targets | 33 ++++++++++++++++++++++++++ bin/report-targets2vars | 52 +++++++++++++++++++++++++++++++++++++++++ doc/assumptions.txt | 6 +++++ doc/variables.txt | 5 ++++ lib/build.mk | 7 ++++-- lib/clean.mk | 23 +++++++++++------- lib/profile.mk | 43 ++++++++++++++++++++-------------- lib/report.mk | 9 +++++++ reports.mk | 14 +++++++++++ 11 files changed, 185 insertions(+), 36 deletions(-) create mode 100755 bin/report-filter create mode 100755 bin/report-targets create mode 100755 bin/report-targets2vars create mode 100644 lib/report.mk create mode 100644 reports.mk diff --git a/Makefile b/Makefile index d5f4a7cf..dbebf4ff 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ # umbrella mkimage-profiles makefile: -# iterate over multiple goals/arches +# iterate over multiple goals/arches, +# collect proceedings -# immediate assignment +# for immediate assignment ifndef ARCHES ifdef ARCH ARCHES := $(ARCH) @@ -11,8 +12,14 @@ endif endif export ARCHES +# supervise target tracing; leave stderr alone +ifdef REPORT +export REPORT_PATH := $(shell mktemp --tmpdir mkimage-profiles.report.XXXXXXX) +POSTPROC := | bin/report-filter > $(REPORT_PATH) +endif + # recursive make considered useful for m-p -MAKE += --no-print-directory +MAKE += -r --no-print-directory .PHONY: clean distclean help clean distclean help: @@ -23,16 +30,18 @@ export NUM_TARGETS := $(words $(MAKECMDGOALS)) # real targets need real work %: @n=1; \ + say() { echo "$$@" >&2; }; \ if [ "$(NUM_TARGETS)" -gt 1 ]; then \ n="`echo $(MAKECMDGOALS) \ | tr '[[:space:]]' '\n' \ | grep -nx "$@" \ | cut -d: -f1`"; \ - echo "** goal: $@ [$$n/$(NUM_TARGETS)]"; \ + say "** goal: $@ [$$n/$(NUM_TARGETS)]"; \ fi; \ for ARCH in $(ARCHES); do \ - if [ "$$ARCH" != "$(firstword $(ARCHES))" ]; then echo; fi; \ - echo "** ARCH: $$ARCH"; \ - $(MAKE) -f main.mk ARCH=$$ARCH $@; \ + if [ "$$ARCH" != "$(firstword $(ARCHES))" ]; then say; fi; \ + say "** ARCH: $$ARCH" >&2; \ + $(MAKE) -f main.mk ARCH=$$ARCH $@ $(POSTPROC); \ + $(MAKE) -f reports.mk ARCH=$$ARCH; \ done; \ - if [ "$$n" -lt "$(NUM_TARGETS)" ]; then echo; fi + if [ "$$n" -lt "$(NUM_TARGETS)" ]; then say; fi diff --git a/bin/report-filter b/bin/report-filter new file mode 100755 index 00000000..fac5e1b0 --- /dev/null +++ b/bin/report-filter @@ -0,0 +1,4 @@ +#!/bin/sh +# filter worker make stdout for report scripts + +grep '^trace:building ' | uniq diff --git a/bin/report-targets b/bin/report-targets new file mode 100755 index 00000000..bb9e737b --- /dev/null +++ b/bin/report-targets @@ -0,0 +1,33 @@ +#!/bin/sh + +echo "digraph {" +echo " { node [fontname=Helvetica,fontsize=20];" + +while read first second third rest; do + FROM=; TO= + case "$first" in + "trace:building") + case "$third" in + "->") + FROM="$second"; TO="$rest";; + *) + continue;; + esac + ;; + *) + continue;; + esac + [ -n "$FROM" -a -n "$TO" ] || continue + for to in $TO; do + out=" \"$FROM\" -> \"$to\"" + case $to in + *distro/*) + echo "$out [weight=10];";; + *) + echo "$out";; + esac + done +done + +echo " }" +echo "}" diff --git a/bin/report-targets2vars b/bin/report-targets2vars new file mode 100755 index 00000000..b7a2fb10 --- /dev/null +++ b/bin/report-targets2vars @@ -0,0 +1,52 @@ +#!/bin/sh + +DISTCFG=build/distcfg.mk +[ -s "$DISTCFG" ] || exit 1 + +VARIABLES= + +echo "graph { rankdir=LR;" +echo " { node [height=.1,width=.3,fontname=Helvetica,fontsize=10];" + +feat_vars() +{ +FEATURE= + while read first second rest; do + case "$first" in + \#[A-Z]*) + continue;; # overridden feature + \#) # feature mark + case "$second" in + profile/*) + FEATURE=;; + *) + FEATURE="$second";; + esac + ;; + *) + case "$second" in + =|+=|?=) + case "$first" in + DISTCFG_MK|SUBPROFILES|FEATURES|IMAGE*|MKIMAGE_*) + continue;; + *) + VAR="$first" + VARIABLES="$VARIABLES; $VAR" + ;; + esac + ;; + *) + continue;; + esac + ;; + esac + [ -n "$FEATURE" -a -n "$VAR" ] || continue + echo " \"$FEATURE\" -- \"$VAR\";" + done < "$DISTCFG" + echo " { node [shape=box]$VARIABLES; }" +} + +feat_vars | LC_COLLATE=C sort -ru + +echo " }" +echo "}" diff --git a/doc/assumptions.txt b/doc/assumptions.txt index 32833636..9e88425a 100644 --- a/doc/assumptions.txt +++ b/doc/assumptions.txt @@ -3,6 +3,12 @@ NB: пути приводятся от верхнего уровня; проект в целом предполагает GNU make 3.81 (с использованием которого и разрабатывается). +- lib/report.mk + + ожидает, что каждая подлежащая трассированию цель каждого + makefile при сборке конфигурации образа содержит непустой + recipe -- хотя бы "; @:" -- т.к. зависит от запуска $(SHELL) + + характерный признак пропуска -- разрыв графа (report-targets.png) + - pkg.in/lists/Makefile + ожидает, что названия пакаджлистов указываются в переменных вида *_LISTS, и копирует в генерируемый профиль только их diff --git a/doc/variables.txt b/doc/variables.txt index 7ac13518..a54aad5e 100644 --- a/doc/variables.txt +++ b/doc/variables.txt @@ -65,6 +65,11 @@ + значение: пусто (по умолчанию) либо любая строка + см. ../lib/build.mk +- REPORT + + запрашивает создание отчёта о собранном образе + + значение: пусто (по умолчанию) либо любая строка + + см. ../Makefile, ../report.mk, ../lib/report.mk + - SAVE_PROFILE + сохраняет архив сгенерированного профиля в .disk/ + значение: пусто (по умолчанию) либо любая строка diff --git a/lib/build.mk b/lib/build.mk index fa26a564..cd618b61 100644 --- a/lib/build.mk +++ b/lib/build.mk @@ -31,8 +31,10 @@ IMAGEDIR ?= $(shell \ ) # actual build starter +# NB: our output MUST go into stderr to escape POSTPROC build-image: profile/populate - @if [ -n "$(CHECK)" ]; then \ + @{ \ + if [ -n "$(CHECK)" ]; then \ echo "$(TIME) skipping actual image build (CHECK is set)"; \ exit; \ fi; \ @@ -61,4 +63,5 @@ build-image: profile/populate df -P $(BUILDDIR) | awk 'END { if ($$4 < $(LOWSPACE)) \ { print "NB: low space on "$$6" ("$$5" used)"}}'; \ fi; \ - if [ -n "$(BELL)" ]; then echo -ne '\a' >&2; fi + if [ -n "$(BELL)" ]; then echo -ne '\a'; fi; \ + } >&2 diff --git a/lib/clean.mk b/lib/clean.mk index cf33c617..05d779e0 100644 --- a/lib/clean.mk +++ b/lib/clean.mk @@ -19,19 +19,23 @@ endif endif # ordinary clean: destroys workdirs but not the corresponding results +# NB: our output MUST go into stderr to escape POSTPROC clean: - @find -name '*~' -delete >&/dev/null ||: - @if [ -L "$(SYMLINK)" -a -d "$(SYMLINK)"/ ]; then \ + @{ \ + find -name '*~' -delete >&/dev/null ||:; \ + if [ -L "$(SYMLINK)" -a -d "$(SYMLINK)"/ ]; then \ echo "$(TIME) cleaning up $(WARNING)"; \ $(MAKE) -C "$(SYMLINK)" $@ \ GLOBAL_BUILDDIR="$(realpath $(SYMLINK))" $(LOG) ||:; \ - fi + fi; \ + } >&2 # there can be some sense in writing log here even if normally # $(BUILDDIR)/ gets purged: make might have failed, # and BUILDLOG can be specified by hand either distclean: clean - @if [ -L "$(SYMLINK)" -a -d "$(SYMLINK)"/ ]; then \ + @{ \ + if [ -L "$(SYMLINK)" -a -d "$(SYMLINK)"/ ]; then \ build="$(realpath $(SYMLINK)/)"; \ if [ "$$build" = / ]; then \ echo "** ERROR: invalid \`"$(SYMLINK)"' symlink" >&2; \ @@ -41,16 +45,19 @@ distclean: clean GLOBAL_BUILDDIR="$$build" $(LOG) ||: \ rm -rf "$$build"; \ fi; \ - fi - @rm -f "$(SYMLINK)" + fi; \ + rm -f "$(SYMLINK)"; \ + } >&2 # builddir existing outside read-only metaprofile is less ephemeral # than BUILDDIR is -- usually it's unneeded afterwards so just zap it postclean: build-image - @if [ "$(NUM_TARGETS)" -gt 1 -a -z "$(DEBUG)" ] || \ + @{ \ + if [ "$(NUM_TARGETS)" -gt 1 -a -z "$(DEBUG)" ] || \ [ ! -L "$(SYMLINK)" -a "0$(DEBUG)" -lt 2 ]; then \ echo "$(TIME) cleaning up after build"; \ $(MAKE) -C "$(BUILDDIR)" distclean \ GLOBAL_BUILDDIR="$(BUILDDIR)" $(LOG) ||:; \ rm -rf "$(BUILDDIR)"; \ - fi + fi; \ + } >&2 diff --git a/lib/profile.mk b/lib/profile.mk index a5ab6b29..a868635d 100644 --- a/lib/profile.mk +++ b/lib/profile.mk @@ -34,50 +34,57 @@ CONFIG := $(BUILDDIR)/distcfg.mk RC := $(HOME)/.mkimage/profiles.mk # step 1: initialize the off-tree mkimage profile (BUILDDIR) +# NB: our output MUST go into stderr to escape POSTPROC profile/init: distclean - @if [ "`realpath "$(BUILDDIR)/"`" = / ]; then \ + @{ \ + if [ "`realpath "$(BUILDDIR)/"`" = / ]; then \ echo "$(TIME) ERROR: invalid BUILDDIR: \`$(BUILDDIR)'"; \ exit 128; \ - fi; - @echo -n "$(TIME) initializing BUILDDIR: " - @rsync -qaxH --delete-after image.in/ "$(BUILDDIR)"/ - @mkdir "$(BUILDDIR)"/.mki # mkimage toplevel marker + fi; \ + echo -n "$(TIME) initializing BUILDDIR: "; \ + rsync -qaxH --delete-after image.in/ "$(BUILDDIR)"/; \ + mkdir "$(BUILDDIR)"/.mki; \ + } >&2 @$(call put,ifndef DISTCFG_MK) @$(call put,DISTCFG_MK = 1) - @if type -t git >&/dev/null; then \ + @{ \ + if type -t git >&/dev/null; then \ if [ -d .git ]; then \ git show-ref --head -d -s -- HEAD && \ git status -s && \ echo; \ fi $(LOG); \ - fi - @{ \ + fi; \ + { \ eval `apt-config shell $${APTCONF:+-c=$(wildcard $(APTCONF))} \ SOURCELIST Dir::Etc::sourcelist/f \ SOURCEPARTS Dir::Etc::sourceparts/d`; \ find "$$SOURCEPARTS" -name '*.list' \ | xargs egrep -Rhv '^#|^[[:blank:]]*$$' "$$SOURCELIST" && \ echo; \ - } $(LOG) - @if type -t git >&/dev/null; then \ + } $(LOG); \ + if type -t git >&/dev/null; then \ if cd $(BUILDDIR); then \ git init -q && \ git add . && \ git commit -qam 'derivative profile initialized'; \ + cd ->&/dev/null; \ fi; \ - fi - @if [ -w . ]; then \ + fi; \ + if [ -w . ]; then \ rm -f "$(SYMLINK)" && \ ln -s "$(BUILDDIR)" "$(SYMLINK)" && \ echo "$(SYMLINK)/"; \ else \ - echo "$(BUILDDIR)/" $(SHORTEN); \ - fi $(SHORTEN) + echo "$(BUILDDIR)/"; \ + fi $(SHORTEN); \ + } >&2 profile/bare: profile/init - @NOTE="$${GLOBAL_VERBOSE:+: $(CONFIG)}"; \ - echo "$(TIME) preparing distro config$$NOTE" \ - $(SHORTEN) + @{ \ + NOTE="$${GLOBAL_VERBOSE:+: $(CONFIG)}"; \ + echo "$(TIME) preparing distro config$$NOTE" $(SHORTEN); \ + } >&2 @$(call try,MKIMAGE_PREFIX,/usr/share/mkimage) @$(call try,GLOBAL_VERBOSE,) @$(call try,IMAGEDIR,$(IMAGEDIR)) @@ -106,7 +113,7 @@ profile/dump-vars: fi $(LOG) # step 3 entry point: copy the needed parts into BUILDDIR -profile/populate: profile/init profile/finalize profile/dump-vars +profile/populate: profile/finalize profile/dump-vars @for dir in sub.in features.in pkg.in; do \ $(MAKE) -C $$dir $(LOG); \ done diff --git a/lib/report.mk b/lib/report.mk new file mode 100644 index 00000000..21580c31 --- /dev/null +++ b/lib/report.mk @@ -0,0 +1,9 @@ +# enable make target tracing + +ifdef REPORT +TRACE_PREFIX := trace:building +OLD_SHELL := $(SHELL) +SHELL = $(info $(TRACE_PREFIX) $@$(if $^$|, -> $^ $|))$(OLD_SHELL) +# piggyback BUILDDIR back into supervising environment +$(info $(TRACE_PREFIX) BUILDDIR = $(BUILDDIR)) +endif diff --git a/reports.mk b/reports.mk new file mode 100644 index 00000000..13dbcef5 --- /dev/null +++ b/reports.mk @@ -0,0 +1,14 @@ +# collect what's left + +all: reports/targets + +reports/targets: + @if [ -n "$$REPORT_PATH" -a -s "$$REPORT_PATH" ]; then \ + BUILDDIR="`sed -n 's/^.* BUILDDIR = \(.*\)/\1/p' \ + "$$REPORT_PATH"`"; \ + REPORT_IMAGE="$$BUILDDIR/targets.png"; \ + bin/report-targets < "$$REPORT_PATH" \ + | dot -Tpng -o "$$REPORT_IMAGE" \ + && echo "** target graph report: $$REPORT_IMAGE" \ + && mv "$$REPORT_PATH" "$$BUILDDIR/targets.log"; \ + fi