From fde591f2c410fe028c71120addb80cc147a5ec86 Mon Sep 17 00:00:00 2001 From: Tony Theodore Date: Thu, 27 Nov 2014 17:58:42 +1100 Subject: Makefile: build targets in parallel I was never really happy with the way targets are serialized, it's fighting against `make` (causing issues like #234 and #219), and the target header isn't particularly informative. This fixes #234 and should also provide a speedup when building all targets simultaneously. Download rules are now pre-requisites of build rules to stop multiple download attempts, and all build logs have a target suffix. More work is needed on #219, since download rules are always outdated unless a marker file is used. I'll look at this separately as I also want to allow script only packages. --- Makefile | 84 ++++++++++++++++++++++------------------------------------------ 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 7352df6..3e16909 100644 --- a/Makefile +++ b/Makefile @@ -207,8 +207,12 @@ else } >'$(PWD)/settings.mk') endif +# Numeric min and max list functions +LIST_NMAX = $(shell echo '$(strip $(1))' | tr ' ' '\n' | sort -n | tail -1) +LIST_NMIN = $(shell echo '$(strip $(1))' | tr ' ' '\n' | sort -n | head -1) + NPROCS := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1) -JOBS_AUTO := $(shell printf "$(DEFAULT_MAX_JOBS)\n$(NPROCS)" | sort -n | head -1) +JOBS_AUTO := $(call LIST_NMIN, $(DEFAULT_MAX_JOBS) $(NPROCS)) JOBS := $(strip $(if $(findstring undefined,$(origin JOBS)),\ $(info [using autodetected $(JOBS_AUTO) job(s)]) \ $(JOBS_AUTO)\ @@ -284,6 +288,12 @@ BUILD_PKGS := $(call set_create, \ $(SED) -n 's,.*src/\(.*\)\.mk,\1,p'), \ $(if $(value $(call LOOKUP_PKG_RULE,$(PKG),BUILD,$(BUILD))), $(PKG)))) +# set column widths for build status messages +PKG_COL_WIDTH := $(call plus,2,$(call LIST_NMAX, $(sort $(call map, strlen, $(PKGS))))) +MAX_TARGET_WIDTH := $(call LIST_NMAX, $(sort $(call map, strlen, $(MXE_TARGETS)))) +TARGET_COL_WIDTH := $(call subtract,100,$(call plus,$(PKG_COL_WIDTH),$(MAX_TARGET_WIDTH))) +PRINTF_FMT := printf '%-11s %-$(PKG_COL_WIDTH)s %-$(TARGET_COL_WIDTH)s %-15s %s\n' + .PHONY: download download: $(addprefix download-,$(PKGS)) @@ -291,31 +301,7 @@ download: $(addprefix download-,$(PKGS)) build-requirements: @$(MAKE) -f '$(MAKEFILE)' $(BUILD_PKGS) MXE_TARGETS=$(BUILD) DONT_CHECK_REQUIREMENTS=true -define TARGET_DEPS -$(1)_DEPS := $(shell echo '$(MXE_TARGETS)' | \ - $(SED) -n 's,.*$(1)\(.*\),\1,p' | \ - awk '{print $$1}') -endef -$(foreach TARGET,$(MXE_TARGETS),$(eval $(call TARGET_DEPS,$(TARGET)))) - -TARGET_HEADER = \ - $(strip with \ - $(if $(value MAKECMDGOALS),\ - $(words $(MAKECMDGOALS)) goal$(shell [ $(words $(MAKECMDGOALS)) == 1 ] || echo s) from command line,\ - $(if $(value LOCAL_PKG_LIST),\ - $(words $(LOCAL_PKG_LIST)) goal$(shell [ $(words $(LOCAL_PKG_LIST)) == 1 ] || echo s) from settings.mk,\ - $(words $(PKGS)) goal$(shell [ $(words $(PKGS)) == 1 ] || echo s) from src/*.mk))) - define TARGET_RULE -.PHONY: $(1) -$(1): | $(if $(value $(1)_DEPS), \ - $(if $(value MAKECMDGOALS),\ - $(addprefix $(PREFIX)/$($(1)_DEPS)/installed/,$(MAKECMDGOALS)), \ - $(if $(value LOCAL_PKG_LIST),\ - $(addprefix $(PREFIX)/$($(1)_DEPS)/installed/,$(LOCAL_PKG_LIST)), \ - $(addprefix $(PREFIX)/$($(1)_DEPS)/installed/,$(PKGS))))) \ - $($(1)_DEPS) - @echo '[target] $(1) $(call TARGET_HEADER)' $(if $(findstring i686-pc-mingw32,$(1)), $(error Deprecated target specified: "$(1)". Please use \ i686-w64-mingw32.[$(subst $(space),|,$(MXE_LIB_TYPES))] instead)) @@ -335,14 +321,14 @@ $(foreach TARGET,$(MXE_TARGETS),$(eval $(call TARGET_RULE,$(TARGET)))) define PKG_RULE .PHONY: download-$(1) -download-$(1):: $(addprefix download-,$(value $(call LOOKUP_PKG_RULE,$(1),DEPS,$(3)))) \ +download-$(1): $(addprefix download-,$(value $(call LOOKUP_PKG_RULE,$(1),DEPS,$(3)))) \ download-only-$(1) .PHONY: download-only-$(1) -download-only-$(1):: +download-only-$(1): @[ -d '$(LOG_DIR)/$(TIMESTAMP)' ] || mkdir -p '$(LOG_DIR)/$(TIMESTAMP)' @if ! $(call CHECK_PKG_ARCHIVE,$(1)); then \ - echo '[download] $(1)'; \ + $(PRINTF_FMT) '[download]' '$(1)'; \ ($(call DOWNLOAD_PKG_ARCHIVE,$(1))) &> '$(LOG_DIR)/$(TIMESTAMP)/$(1)-download'; \ ln -sf '$(TIMESTAMP)/$(1)-download' '$(LOG_DIR)/$(1)-download'; \ if ! $(call CHECK_PKG_ARCHIVE,$(1)); then \ @@ -358,47 +344,33 @@ download-only-$(1):: exit 1; \ fi; \ fi +endef +$(foreach PKG,$(PKGS),$(eval $(call PKG_RULE,$(PKG)))) +define PKG_TARGET_RULE .PHONY: $(1) $(1): $(PREFIX)/$(3)/installed/$(1) $(PREFIX)/$(3)/installed/$(1): $(TOP_DIR)/src/$(1).mk \ $(wildcard $(TOP_DIR)/src/$(1)-*.patch) \ $(wildcard $(TOP_DIR)/src/$(1)-test*) \ $(addprefix $(PREFIX)/$(3)/installed/,$(value $(call LOOKUP_PKG_RULE,$(1),DEPS,$(3)))) \ - | $(if $(DONT_CHECK_REQUIREMENTS),,check-requirements) $(3) + | $(if $(DONT_CHECK_REQUIREMENTS),,check-requirements) \ + download-only-$(1) @[ -d '$(LOG_DIR)/$(TIMESTAMP)' ] || mkdir -p '$(LOG_DIR)/$(TIMESTAMP)' - @if ! $(call CHECK_PKG_ARCHIVE,$(1)); then \ - echo '[download] $(1)'; \ - ($(call DOWNLOAD_PKG_ARCHIVE,$(1))) &> '$(LOG_DIR)/$(TIMESTAMP)/$(1)-download'; \ - ln -sf '$(TIMESTAMP)/$(1)-download' '$(LOG_DIR)/$(1)-download'; \ - if ! $(call CHECK_PKG_ARCHIVE,$(1)); then \ - echo; \ - echo 'Download failed or wrong checksum of package $(1)!'; \ - echo '------------------------------------------------------------'; \ - $(if $(findstring undefined, $(origin MXE_VERBOSE)),\ - tail -n 10 '$(LOG_DIR)/$(1)-download' | $(SED) -n '/./p';, \ - $(SED) -n '/./p' '$(LOG_DIR)/$(1)-download';) \ - echo '------------------------------------------------------------'; \ - echo '[log] $(LOG_DIR)/$(1)-download'; \ - echo; \ - exit 1; \ - fi; \ - fi $(if $(value $(call LOOKUP_PKG_RULE,$(1),BUILD,$(3))), - @echo '[build] $(1)', - @echo '[no-build] $(1)') + @$(PRINTF_FMT) '[build]' '$(1)' '$(3)', + @$(PRINTF_FMT) '[no-build]' '$(1)' '$(3)') @touch '$(LOG_DIR)/$(TIMESTAMP)/$(1)_$(3)' @ln -sf '$(TIMESTAMP)/$(1)_$(3)' '$(LOG_DIR)/$(1)_$(3)' - @ln -sf '$(TIMESTAMP)/$(1)_$(3)' '$(LOG_DIR)/$(1)' @if ! (time $(MAKE) -f '$(MAKEFILE)' 'build-only-$(1)_$(3)') &> '$(LOG_DIR)/$(TIMESTAMP)/$(1)_$(3)'; then \ echo; \ - echo 'Failed to build package $(1)!'; \ + echo 'Failed to build package $(1) for target $(3)!'; \ echo '------------------------------------------------------------'; \ $(if $(findstring undefined, $(origin MXE_VERBOSE)),\ - tail -n 10 '$(LOG_DIR)/$(1)' | $(SED) -n '/./p';, \ - $(SED) -n '/./p' '$(LOG_DIR)/$(1)';) \ + tail -n 10 '$(LOG_DIR)/$(1)_$(3)' | $(SED) -n '/./p';, \ + $(SED) -n '/./p' '$(LOG_DIR)/$(1)_$(3)';) \ echo '------------------------------------------------------------'; \ - echo '[log] $(LOG_DIR)/$(1)'; \ + echo '[log] $(LOG_DIR)/$(1)_$(3)'; \ echo; \ (echo; \ find '$(2)' -name 'config.log' -print -exec cat {} \;; \ @@ -409,7 +381,9 @@ $(PREFIX)/$(3)/installed/$(1): $(TOP_DIR)/src/$(1).mk \ exit 1; \ fi $(if $(value $(call LOOKUP_PKG_RULE,$(1),BUILD,$(3))), - @echo '[done] $(1)') + @$(PRINTF_FMT) '[done]' '$(1)' '$(3)' "`grep -a '^du:.*KiB$$\' '$(LOG_DIR)/$(TIMESTAMP)/$(1)_$(3)' | cut -d ':' -f2`" \ + "`grep -a '^real.*m.*s$$\' '$(LOG_DIR)/$(TIMESTAMP)/$(1)_$(3)' | tr '\t' ' ' | cut -d ' ' -f2`") + .PHONY: build-only-$(1)_$(3) build-only-$(1)_$(3): PKG = $(1) @@ -438,7 +412,7 @@ endef $(foreach TARGET,$(MXE_TARGETS), \ $(shell [ -d '$(PREFIX)/$(TARGET)/installed' ] || mkdir -p '$(PREFIX)/$(TARGET)/installed') \ $(foreach PKG,$(PKGS), \ - $(eval $(call PKG_RULE,$(PKG),$(call TMP_DIR,$(PKG)-$(TARGET)),$(TARGET))))) + $(eval $(call PKG_TARGET_RULE,$(PKG),$(call TMP_DIR,$(PKG)-$(TARGET)),$(TARGET))))) # convenience set-like functions for unique lists SET_APPEND = \ -- cgit v0.12