From 38c43b5017befbf0153d3a5f2165e80c064329ad Mon Sep 17 00:00:00 2001 From: Tony Theodore Date: Sun, 11 Mar 2018 13:19:24 +1100 Subject: sub makefiles: standardize names and add to build-pkg --- Makefile | 6 +- github.mk | 186 ---------------------------------------------------- mxe.github.mk | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++ mxe.patch.mk | 126 +++++++++++++++++++++++++++++++++++ mxe.updates.mk | 79 ++++++++++++++++++++++ patch.mk | 126 ----------------------------------- tools/build-pkg.lua | 10 +-- updates.mk | 79 ---------------------- 8 files changed, 400 insertions(+), 398 deletions(-) delete mode 100644 github.mk create mode 100644 mxe.github.mk create mode 100644 mxe.patch.mk create mode 100644 mxe.updates.mk delete mode 100644 patch.mk delete mode 100644 updates.mk diff --git a/Makefile b/Makefile index 3438a36..4318c13 100644 --- a/Makefile +++ b/Makefile @@ -215,7 +215,7 @@ define AUTOTOOLS_BUILD endef # include github related functions -include $(PWD)/github.mk +include $(PWD)/mxe.github.mk # shared lib preload to disable networking, enable faketime etc PRELOAD_VARS := LD_PRELOAD DYLD_FORCE_FLAT_NAMESPACE DYLD_INSERT_LIBRARIES @@ -1081,5 +1081,5 @@ docs/packages.json: $(foreach 1,$(PKGS),$(PKG_MAKEFILES)) @echo '}' >> $@ # for other mxe functions -include patch.mk -include updates.mk +include mxe.patch.mk +include mxe.updates.mk diff --git a/github.mk b/github.mk deleted file mode 100644 index e2bbd1a..0000000 --- a/github.mk +++ /dev/null @@ -1,186 +0,0 @@ -# This file is part of MXE. See LICENSE.md for licensing information. - -# Standardise GitHub downloads and updates -# Download API has three forms: -# Archive: -# url = //archive/.tar.gz -# dir = - -# if starts with a single `v`, it is removed from dir -# -# Release: -# Manually uploaded distribution tarballs, especially useful for -# autotools packages with generated sources. No universal convention, -# but generally: -# url = //releases/downloads//-.tar.[bz2,gz,xz,...] -# dir = - -# -# Tarball: -# url = //tarball//output-file.tar.gz -# dir = -- -# -# TODO: update remaining packages -# grep -l 'MXE_GET_GITHUB\|api.github.com\|github.com.*' src/*.mk | xargs grep -L 'GH_CONF' -# -# Filename doesn't matter as we stream the url to a name of our choosing. -# -# The archive API could be used for all packages, however, if the reference -# is any sort of SHA, the full SHA is used for the directory. We could `cd` -# into it without knowing the SHA beforehand, but the directory length would -# be comical in logs etc. -# -# The release API is based on tags but the uploaded tarballs may use -# any naming convention for the filename and subdir, and also other -# archive types e.g. *.xz -# -# The tarball API accepts references to commits or tags, always using the -# short SHA as the directory. In this case, tag tracking packages would have -# to store the SHA (see #1002). However, this only works for lightweight -# tags, not annotated tags that most projects use for releases. -# -# In summary, we have to use all three. - - -# The tarball API determines the short SHA length used in the directory name. -# Chances of a collision on a "given" commit seem to decrease as the chance -# of "any" collision increases. If that changes in the future, `make update` -# will fix it. -# Currently (2015-12) the API sets the short SHA length to: -GITHUB_SHA_LENGTH := 7 - -# Packages must set the following metadata: -# Track branch - Tarball API -# GH_CONF := owner/repo/branch -# updates will use the last commit from the specified branch as -# a version string and bypass `sort -V` -# -# Track releases - Release API -# GH_CONF := owner/repo/releases[/latest], tag prefix, tag suffix, tag filter-out, version separator -# updates can optionally use the latest non-prerelease with /latest -# or manually specify version numbering based on: -# /./version> -# -# Track tags - Archive API -# GH_CONF := owner/repo/tags, tag prefix, tag suffix, tag filter-out, version separator -# updates will construct a version number based on: -# /./version> -# -# Using the third segment for api means you can't track a branch with these names: -GH_APIS := releases tags - -# common tag filtering is applied with `grep -v`: -GITHUB_TAG_FILTER := alpha\|beta\|rc - -# More complex filters can fall back to `MXE_GET_GH_TAGS` which returns -# a list for post-processing. - -# ...and finally, auto-configure packages based on above metadata: -# - `eval` these snippets during PKG_RULE loop (where PKG is in scope). -# - `call` or `eval` from package makefiles requires complex quoting -# and looks out of place. -# - don't redefine manually set standard variables (FILE, SUBDIR, URL, UPDATE) - -GH_OWNER = $(word 1,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) -GH_REPO = $(word 2,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) -GH_API = $(filter $(GH_APIS),$(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))) -GH_BRANCH = $(filter-out $(GH_APIS),$(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))) -GH_LATEST = $(word 4,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) -GH_TAG_VARS = $(call rest,$(subst $(comma),$(space)$(__gmsl_aa_magic),$(subst $(space),,$($(PKG)_GH_CONF)))) -GH_TAG_PREFIX = $(subst $(__gmsl_aa_magic),,$(word 1,$(GH_TAG_VARS))) -GH_TAG_SUFFIX = $(subst $(__gmsl_aa_magic),,$(word 2,$(GH_TAG_VARS))) -GH_TAG_FILTER = $(subst $(__gmsl_aa_magic),,$(word 3,$(GH_TAG_VARS))) -GH_VERSION_SEP = $(subst $(__gmsl_aa_magic),,$(word 4,$(GH_TAG_VARS))) - -define MXE_SETUP_GITHUB - $(PKG)_GH_OWNER := $(GH_OWNER) - $(PKG)_GH_REPO := $(GH_REPO) - $(PKG)_GH_LATEST := $(if $(GH_LATEST),/latest) - $(PKG)_BRANCH := $(GH_BRANCH) - $(PKG)_TAG_VARS := $(GH_TAG_VARS) - $(PKG)_TAG_PREFIX := $(GH_TAG_PREFIX) - $(PKG)_TAG_SUFFIX := $(GH_TAG_SUFFIX) - $(PKG)_TAG_FILTER := $(GH_TAG_FILTER) - $(PKG)_VERSION_SEP := $(or $(GH_VERSION_SEP),.) - $(PKG)_FILE := $(or $($(PKG)_FILE),$(PKG)-$$(filter-out $$(PKG)-,$$($$(PKG)_TAG_PREFIX))$($(PKG)_VERSION)$$($$(PKG)_TAG_SUFFIX).tar.gz) - $(if $(and $(GH_BRANCH),$(GH_TAG_VARS)),\ - $(error $(newline) $(PKG) specifies both branch and tag variables $(newline))) - $(if $(and $(GH_BRANCH),$(GH_LATEST)),\ - $(error $(newline) $(PKG) has fragments after github branch $(newline))) - $(if $(GH_BRANCH),$(value MXE_SETUP_GITHUB_BRANCH),$(value MXE_SETUP_GITHUB_$(call uc,$(GH_API)))) -endef - -define MXE_SETUP_GITHUB_BRANCH - $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_OWNER)-$($(PKG)_GH_REPO)-$($(PKG)_VERSION)) - $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE)) - $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_SHA,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_BRANCH))) -endef - -define MXE_SETUP_GITHUB_RELEASES - $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) - $(PKG)_TAG_REF := $(or $($(PKG)_TAG_REF),$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) - $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases/download/$($(PKG)_TAG_REF)/$($(PKG)_SUBDIR).tar.gz) - $(PKG)_URL_2 := $(or $($(PKG)_URL_2),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$($(PKG)_TAG_REF).tar.gz) - $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_RELEASE,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases$($(PKG)_GH_LATEST),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP))) -endef - -define MXE_SETUP_GITHUB_TAGS - $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) - $(PKG)_TAR_GZ := $(or $($(PKG)_TAR_GZ),$($(PKG)_GH_REPO)-$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) - $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$(subst $($(PKG)_GH_REPO)-,,$($(PKG)_TAR_GZ)).tar.gz) - $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_TAG,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP))) -endef - -# called with owner/repo/releases[/latest],tag prefix, tag suffix, filter-out, version sep -define MXE_GET_GH_RELEASE - $(WGET) -q -O- 'https://github.com/$(strip $(1))' \ - | $(SED) -n 's,.*releases/tag/\([^"]*\)".*,\1,p' \ - | $(if $(4),grep -vi '$(strip $(4))') \ - | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \ - | tr '$(strip $(5))' '.' \ - | $(SORT) -V \ - | tail -1 -endef - -# called with owner/repo,branch -define MXE_GET_GH_SHA - $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/heads/$(strip $(2))' \ - | $(SED) -n 's#.*"sha": "\([^"]\{$(GITHUB_SHA_LENGTH)\}\).*#\1#p' \ - | head -1 -endef - -# called with owner/repo -define MXE_GET_GH_TAGS - $(WGET) -q -O- 'https://github.com/$(strip $(1))/tags' \ - | $(SED) -n 's#.*releases/tag/\([^"]*\).*#\1#p' -endef - -# called with owner/repo, tag prefix, tag suffix, filter-out, version sep -define MXE_GET_GH_TAG - $(MXE_GET_GH_TAGS) \ - | $(if $(4),grep -vi '$(strip $(4))') \ - | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \ - | tr '$(strip $(5))' '.' \ - | $(SORT) -V \ - | tail -1 -endef - -GITHUB_PKGS = $(patsubst %_GH_CONF,%,$(filter %_GH_CONF,$(.VARIABLES))) - -# check-gh-conf : test updates and source directory -# check-gh-conf-dl: removes downloads and tests above - -# a test of many package updates may hit rate limit of 60/hr -# https://developer.github.com/v3/#rate-limiting - -.PHONY: check-gh-conf check-gh-conf-% -check-gh-conf-dl: REMOVE_DOWNLOAD = true -check-gh-conf-dl: MXE_NO_BACKUP_DL = true -check-gh-conf-dl: check-gh-conf -check-gh-conf-pkg-%: check-update-package-% download-only-% - @$(PRINTF_FMT) '[prep-src]' '$(*)' | $(RTRIM) - @($(MAKE) -f '$(MAKEFILE)' 'prepare-pkg-source-$(*)') > /dev/null - @rm -rf '$(call TMP_DIR,$(*))' - -# secondexpansion here since this file is included before pkg makefiles -.SECONDEXPANSION: -check-gh-conf: $$(addprefix check-gh-conf-pkg-,$$(GITHUB_PKGS)) -github-pkgs: $$(GITHUB_PKGS) diff --git a/mxe.github.mk b/mxe.github.mk new file mode 100644 index 0000000..e2bbd1a --- /dev/null +++ b/mxe.github.mk @@ -0,0 +1,186 @@ +# This file is part of MXE. See LICENSE.md for licensing information. + +# Standardise GitHub downloads and updates +# Download API has three forms: +# Archive: +# url = //archive/.tar.gz +# dir = - +# if starts with a single `v`, it is removed from dir +# +# Release: +# Manually uploaded distribution tarballs, especially useful for +# autotools packages with generated sources. No universal convention, +# but generally: +# url = //releases/downloads//-.tar.[bz2,gz,xz,...] +# dir = - +# +# Tarball: +# url = //tarball//output-file.tar.gz +# dir = -- +# +# TODO: update remaining packages +# grep -l 'MXE_GET_GITHUB\|api.github.com\|github.com.*' src/*.mk | xargs grep -L 'GH_CONF' +# +# Filename doesn't matter as we stream the url to a name of our choosing. +# +# The archive API could be used for all packages, however, if the reference +# is any sort of SHA, the full SHA is used for the directory. We could `cd` +# into it without knowing the SHA beforehand, but the directory length would +# be comical in logs etc. +# +# The release API is based on tags but the uploaded tarballs may use +# any naming convention for the filename and subdir, and also other +# archive types e.g. *.xz +# +# The tarball API accepts references to commits or tags, always using the +# short SHA as the directory. In this case, tag tracking packages would have +# to store the SHA (see #1002). However, this only works for lightweight +# tags, not annotated tags that most projects use for releases. +# +# In summary, we have to use all three. + + +# The tarball API determines the short SHA length used in the directory name. +# Chances of a collision on a "given" commit seem to decrease as the chance +# of "any" collision increases. If that changes in the future, `make update` +# will fix it. +# Currently (2015-12) the API sets the short SHA length to: +GITHUB_SHA_LENGTH := 7 + +# Packages must set the following metadata: +# Track branch - Tarball API +# GH_CONF := owner/repo/branch +# updates will use the last commit from the specified branch as +# a version string and bypass `sort -V` +# +# Track releases - Release API +# GH_CONF := owner/repo/releases[/latest], tag prefix, tag suffix, tag filter-out, version separator +# updates can optionally use the latest non-prerelease with /latest +# or manually specify version numbering based on: +# /./version> +# +# Track tags - Archive API +# GH_CONF := owner/repo/tags, tag prefix, tag suffix, tag filter-out, version separator +# updates will construct a version number based on: +# /./version> +# +# Using the third segment for api means you can't track a branch with these names: +GH_APIS := releases tags + +# common tag filtering is applied with `grep -v`: +GITHUB_TAG_FILTER := alpha\|beta\|rc + +# More complex filters can fall back to `MXE_GET_GH_TAGS` which returns +# a list for post-processing. + +# ...and finally, auto-configure packages based on above metadata: +# - `eval` these snippets during PKG_RULE loop (where PKG is in scope). +# - `call` or `eval` from package makefiles requires complex quoting +# and looks out of place. +# - don't redefine manually set standard variables (FILE, SUBDIR, URL, UPDATE) + +GH_OWNER = $(word 1,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) +GH_REPO = $(word 2,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) +GH_API = $(filter $(GH_APIS),$(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))) +GH_BRANCH = $(filter-out $(GH_APIS),$(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))) +GH_LATEST = $(word 4,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF))))) +GH_TAG_VARS = $(call rest,$(subst $(comma),$(space)$(__gmsl_aa_magic),$(subst $(space),,$($(PKG)_GH_CONF)))) +GH_TAG_PREFIX = $(subst $(__gmsl_aa_magic),,$(word 1,$(GH_TAG_VARS))) +GH_TAG_SUFFIX = $(subst $(__gmsl_aa_magic),,$(word 2,$(GH_TAG_VARS))) +GH_TAG_FILTER = $(subst $(__gmsl_aa_magic),,$(word 3,$(GH_TAG_VARS))) +GH_VERSION_SEP = $(subst $(__gmsl_aa_magic),,$(word 4,$(GH_TAG_VARS))) + +define MXE_SETUP_GITHUB + $(PKG)_GH_OWNER := $(GH_OWNER) + $(PKG)_GH_REPO := $(GH_REPO) + $(PKG)_GH_LATEST := $(if $(GH_LATEST),/latest) + $(PKG)_BRANCH := $(GH_BRANCH) + $(PKG)_TAG_VARS := $(GH_TAG_VARS) + $(PKG)_TAG_PREFIX := $(GH_TAG_PREFIX) + $(PKG)_TAG_SUFFIX := $(GH_TAG_SUFFIX) + $(PKG)_TAG_FILTER := $(GH_TAG_FILTER) + $(PKG)_VERSION_SEP := $(or $(GH_VERSION_SEP),.) + $(PKG)_FILE := $(or $($(PKG)_FILE),$(PKG)-$$(filter-out $$(PKG)-,$$($$(PKG)_TAG_PREFIX))$($(PKG)_VERSION)$$($$(PKG)_TAG_SUFFIX).tar.gz) + $(if $(and $(GH_BRANCH),$(GH_TAG_VARS)),\ + $(error $(newline) $(PKG) specifies both branch and tag variables $(newline))) + $(if $(and $(GH_BRANCH),$(GH_LATEST)),\ + $(error $(newline) $(PKG) has fragments after github branch $(newline))) + $(if $(GH_BRANCH),$(value MXE_SETUP_GITHUB_BRANCH),$(value MXE_SETUP_GITHUB_$(call uc,$(GH_API)))) +endef + +define MXE_SETUP_GITHUB_BRANCH + $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_OWNER)-$($(PKG)_GH_REPO)-$($(PKG)_VERSION)) + $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE)) + $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_SHA,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_BRANCH))) +endef + +define MXE_SETUP_GITHUB_RELEASES + $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) + $(PKG)_TAG_REF := $(or $($(PKG)_TAG_REF),$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) + $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases/download/$($(PKG)_TAG_REF)/$($(PKG)_SUBDIR).tar.gz) + $(PKG)_URL_2 := $(or $($(PKG)_URL_2),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$($(PKG)_TAG_REF).tar.gz) + $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_RELEASE,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/releases$($(PKG)_GH_LATEST),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP))) +endef + +define MXE_SETUP_GITHUB_TAGS + $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$($(PKG)_GH_REPO)-$(if $(call sne,v,$($(PKG)_TAG_PREFIX)),$($(PKG)_TAG_PREFIX))$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) + $(PKG)_TAR_GZ := $(or $($(PKG)_TAR_GZ),$($(PKG)_GH_REPO)-$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX)) + $(PKG)_URL := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO)/archive/$(subst $($(PKG)_GH_REPO)-,,$($(PKG)_TAR_GZ)).tar.gz) + $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_TAG,$($(PKG)_GH_OWNER)/$($(PKG)_GH_REPO),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP))) +endef + +# called with owner/repo/releases[/latest],tag prefix, tag suffix, filter-out, version sep +define MXE_GET_GH_RELEASE + $(WGET) -q -O- 'https://github.com/$(strip $(1))' \ + | $(SED) -n 's,.*releases/tag/\([^"]*\)".*,\1,p' \ + | $(if $(4),grep -vi '$(strip $(4))') \ + | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \ + | tr '$(strip $(5))' '.' \ + | $(SORT) -V \ + | tail -1 +endef + +# called with owner/repo,branch +define MXE_GET_GH_SHA + $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/heads/$(strip $(2))' \ + | $(SED) -n 's#.*"sha": "\([^"]\{$(GITHUB_SHA_LENGTH)\}\).*#\1#p' \ + | head -1 +endef + +# called with owner/repo +define MXE_GET_GH_TAGS + $(WGET) -q -O- 'https://github.com/$(strip $(1))/tags' \ + | $(SED) -n 's#.*releases/tag/\([^"]*\).*#\1#p' +endef + +# called with owner/repo, tag prefix, tag suffix, filter-out, version sep +define MXE_GET_GH_TAG + $(MXE_GET_GH_TAGS) \ + | $(if $(4),grep -vi '$(strip $(4))') \ + | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \ + | tr '$(strip $(5))' '.' \ + | $(SORT) -V \ + | tail -1 +endef + +GITHUB_PKGS = $(patsubst %_GH_CONF,%,$(filter %_GH_CONF,$(.VARIABLES))) + +# check-gh-conf : test updates and source directory +# check-gh-conf-dl: removes downloads and tests above + +# a test of many package updates may hit rate limit of 60/hr +# https://developer.github.com/v3/#rate-limiting + +.PHONY: check-gh-conf check-gh-conf-% +check-gh-conf-dl: REMOVE_DOWNLOAD = true +check-gh-conf-dl: MXE_NO_BACKUP_DL = true +check-gh-conf-dl: check-gh-conf +check-gh-conf-pkg-%: check-update-package-% download-only-% + @$(PRINTF_FMT) '[prep-src]' '$(*)' | $(RTRIM) + @($(MAKE) -f '$(MAKEFILE)' 'prepare-pkg-source-$(*)') > /dev/null + @rm -rf '$(call TMP_DIR,$(*))' + +# secondexpansion here since this file is included before pkg makefiles +.SECONDEXPANSION: +check-gh-conf: $$(addprefix check-gh-conf-pkg-,$$(GITHUB_PKGS)) +github-pkgs: $$(GITHUB_PKGS) diff --git a/mxe.patch.mk b/mxe.patch.mk new file mode 100644 index 0000000..3daa878 --- /dev/null +++ b/mxe.patch.mk @@ -0,0 +1,126 @@ +# This file is part of MXE. See LICENSE.md for licensing information. + +GIT_DIR = $(if $(patsubst .,,$($(1)_SUBDIR)) \ + ,$(GITS_DIR)/$($(1)_SUBDIR),$(GITS_DIR)/$(1)) + +GIT_CMD = git \ + --work-tree='$(call GIT_DIR,$(1))' \ + --git-dir='$(call GIT_DIR,$(1))'/.git + +PATCH_NAME = 1-fixes + +# can't use PKG_PATCHES here, because it returns existing patches +# while export-patch creates new patch +PATCH_BY_NAME = $(patsubst %.mk,%-$(2).patch,$($(1)_MAKEFILE)) + +define INIT_GIT + # unpack to gits/tmp/pkg + rm -rf '$(GITS_DIR)/tmp' + mkdir -p '$(GITS_DIR)/tmp/$(1)' + cd '$(GITS_DIR)/tmp/$(1)' && $(call UNPACK_PKG_ARCHIVE,$(1)) + # if PKG_SUBDIR is ".", the following will move gits/tmp/pkg + mv '$(abspath $(GITS_DIR)/tmp/$(1)/$($(1)_SUBDIR))' '$(call GIT_DIR,$(1))' + rm -rf '$(GITS_DIR)/tmp' + # rename existing .git directories if any + find '$(call GIT_DIR,$(1))' -name .git -prune -exec sh -c 'mv "$$0" "$$0"_' {} \; + # initialize git + $(call GIT_CMD,$(1)) init + $(call GIT_CMD,$(1)) add -A + $(call GIT_CMD,$(1)) commit -m "init" + $(call GIT_CMD,$(1)) tag dist +endef + +define IMPORT_PATCH + cd '$(call GIT_DIR,$(1))' \ + && cat '$(2)' \ + | $(SED) '/^From/,$$ !d' \ + | $(SED) s/'^From: MXE'/"From: fix@me"/'g;' \ + | $(call GIT_CMD,$(1)) am --keep-cr ; +endef + +define EXPORT_PATCH + cd '$(call GIT_DIR,$(1))' \ + && ( \ + echo 'This file is part of MXE. See LICENSE.md for licensing information.'; \ + echo ''; \ + echo 'Contains ad hoc patches for cross building.'; \ + echo ''; \ + $(call GIT_CMD,$(1)) format-patch \ + --numbered \ + -p \ + --no-signature \ + --stdout \ + --text \ + -M9 \ + dist..HEAD \ + | $(SED) 's/^From [0-9a-f]\{40\} /From 0000000000000000000000000000000000000000 /' \ + | $(SED) 's/^index .......\.\......../index 1111111..2222222/' \ + ) > '$(PATCH_BY_NAME)' +endef + +_init-git-%: TIMESTAMP = patch +_init-git-%: download-only-% + $(if $(call set_is_member,$*,$(PKGS)), \ + $(if $(wildcard $(call GIT_DIR,$*)), \ + $(error $(call GIT_DIR,$*) already exists), \ + $(call INIT_GIT,$*)), \ + $(error Package $* not found)) + +_import-patch-%: + $(if $(call set_is_member,$*,$(PKGS)), \ + $(if $(wildcard $(call GIT_DIR,$*)), \ + $(call IMPORT_PATCH,$*,$(call PATCH_BY_NAME,$*,$(PATCH_NAME))), \ + $(error $(call GIT_DIR,$*) does not exist)), \ + $(error Package $* not found)) + +_import-all-patches-%: + $(if $(call set_is_member,$*,$(PKGS)), \ + $(if $(wildcard $(call GIT_DIR,$*)), \ + $(foreach PKG_PATCH,$(call PKG_PATCHES,$*), \ + $(call IMPORT_PATCH,$*,$(PKG_PATCH))), \ + $(error $(call GIT_DIR,$*) does not exist)), \ + $(error Package $* not found)) + +_export-patch-%: + $(if $(call set_is_member,$*,$(PKGS)), \ + $(if $(wildcard $(call GIT_DIR,$*)), \ + $(call EXPORT_PATCH,$*,$(PATCH_NAME)), \ + $(error $(call GIT_DIR,$*) does not exist)), \ + $(error Package $* not found)) + +# use .SECONDARY: when refreshing all patches if you don't +# want to unpack everything every time +#.SECONDARY: +init-git-%: $(PREFIX)/installed/patch/init-git-% ; +import-patch-%: $(PREFIX)/installed/patch/import-patch-% ; +import-all-patches-%: $(PREFIX)/installed/patch/import-all-patches-% ; +export-patch-%: $(PREFIX)/installed/patch/export-patch-% ; + +refresh-patch-%: $(PREFIX)/installed/patch/refresh-patch-% ; +$(PREFIX)/installed/patch/refresh-patch-%: + @rm -rf $(PWD)/tmp-patch/$* + @$(MAKE) -f '$(MAKEFILE)' init-git-$* GITS_DIR=$(PWD)/tmp-patch/$* + @$(MAKE) -f '$(MAKEFILE)' import-patch-$* GITS_DIR=$(PWD)/tmp-patch/$* + @$(MAKE) -f '$(MAKEFILE)' export-patch-$* GITS_DIR=$(PWD)/tmp-patch/$* + @# darwin sometimes chokes deleting large git repos + @rm -rf $(PWD)/tmp-patch/$* || sleep 5; rm -rf $(PWD)/tmp-patch/$* + +@mkdir -p '$(dir $@)' + @touch '$@' + +$(PREFIX)/installed/patch/%: + @echo '[$*]' + @[ -d '$(LOG_DIR)/patch' ] || mkdir -p '$(LOG_DIR)/patch' + @(time $(MAKE) -f '$(MAKEFILE)' _$*) &> '$(LOG_DIR)/patch/$*' + +@mkdir -p '$(dir $@)' + @touch '$@' + + +PATCH_FORMAT_PATCHES := $(shell find $(MXE_PLUGIN_DIRS) plugins -name "*-$(PATCH_NAME).patch") +PATCH_FORMAT_PKGS := $(sort $(subst -$(PATCH_NAME),,$(basename $(notdir $(PATCH_FORMAT_PATCHES))))) +PATCH_FORMAT_DIRS := $(sort $(basename $(dir $(PATCH_FORMAT_PATCHES)))) + +.PHONY: refresh-patches +refresh-patches: + @$(MAKE) -f '$(MAKEFILE)' -j '$(JOBS)'\ + $(addprefix refresh-patch-,$(PATCH_FORMAT_PKGS)) \ + MXE_PLUGIN_DIRS='$(PATCH_FORMAT_DIRS)' diff --git a/mxe.updates.mk b/mxe.updates.mk new file mode 100644 index 0000000..9a4455d --- /dev/null +++ b/mxe.updates.mk @@ -0,0 +1,79 @@ +# This file is part of MXE. See LICENSE.md for licensing information. + +# Generic updater +# $(call GET_LATEST_VERSION, base url[, prefix, ext, filter, separator]) +# base url : required page returning list of versions +# e.g https://ftp.gnu.org/gnu/libfoo +# prefix : segment before version +# defaults to lastword of url with dash i.e. `libfoo-` +# ext : segment ending version - default `\.tar` +# filter : `grep -i` filter-out pattern - default alpha\|beta\|rc +# separator: transform char to `.` - typically `_` +# +# test changes with: +# make check-get-latest-version +# +# and update tools/skeleton.py with usage notes + +define GET_LATEST_VERSION + $(WGET) -q -O- '$(strip $(1))' | \ + $(SED) -n 's,.* $(2) ignoring)), + $(info NEW $(1) $($(1)_VERSION) --> $(2)) + $(if $(findstring undefined, $(origin UPDATE_DRYRUN)), + $(SED) -i 's/^\([^ ]*_VERSION *:=\).*/\1 $(2)/' '$($(1)_MAKEFILE)' + $(MAKE) -f '$(MAKEFILE)' 'update-checksum-$(1)' \ + || { $(SED) -i 's/^\([^ ]*_VERSION *:=\).*/\1 $($(1)_VERSION)/' '$($(1)_MAKEFILE)'; \ + exit 1; }))), + $(info Unable to update version number of package $(1) \ + $(newline)$(newline)$($(1)_UPDATE)$(newline))) + +endef +update: + $(foreach PKG,$(PKGS),\ + $(and $($(PKG)_UPDATE),$(call UPDATE,$(PKG),$(shell $($(PKG)_UPDATE))))) + +update-package-%: + $(if $(call set_is_member,$*,$(PKGS)), \ + $(and $($*_UPDATE),$(call UPDATE,$*,$(shell $($*_UPDATE)))), \ + $(error Package $* not found)) + @echo -n + +check-update-package-%: UPDATE_DRYRUN = true +check-update-package-%: update-package-% ; + +update-checksum-%: MXE_NO_BACKUP_DL = true +update-checksum-%: SKIP_CHECHSUM = true +update-checksum-%: + $(if $(call set_is_member,$*,$(PKGS)), \ + $(call DOWNLOAD_PKG_ARCHIVE,$*) && \ + $(SED) -i 's/^\([^ ]*_CHECKSUM *:=\).*/\1 '"`$(call PKG_CHECKSUM,$*)`"'/' '$($*_MAKEFILE)', \ + $(error Package $* not found)) diff --git a/patch.mk b/patch.mk deleted file mode 100644 index 3daa878..0000000 --- a/patch.mk +++ /dev/null @@ -1,126 +0,0 @@ -# This file is part of MXE. See LICENSE.md for licensing information. - -GIT_DIR = $(if $(patsubst .,,$($(1)_SUBDIR)) \ - ,$(GITS_DIR)/$($(1)_SUBDIR),$(GITS_DIR)/$(1)) - -GIT_CMD = git \ - --work-tree='$(call GIT_DIR,$(1))' \ - --git-dir='$(call GIT_DIR,$(1))'/.git - -PATCH_NAME = 1-fixes - -# can't use PKG_PATCHES here, because it returns existing patches -# while export-patch creates new patch -PATCH_BY_NAME = $(patsubst %.mk,%-$(2).patch,$($(1)_MAKEFILE)) - -define INIT_GIT - # unpack to gits/tmp/pkg - rm -rf '$(GITS_DIR)/tmp' - mkdir -p '$(GITS_DIR)/tmp/$(1)' - cd '$(GITS_DIR)/tmp/$(1)' && $(call UNPACK_PKG_ARCHIVE,$(1)) - # if PKG_SUBDIR is ".", the following will move gits/tmp/pkg - mv '$(abspath $(GITS_DIR)/tmp/$(1)/$($(1)_SUBDIR))' '$(call GIT_DIR,$(1))' - rm -rf '$(GITS_DIR)/tmp' - # rename existing .git directories if any - find '$(call GIT_DIR,$(1))' -name .git -prune -exec sh -c 'mv "$$0" "$$0"_' {} \; - # initialize git - $(call GIT_CMD,$(1)) init - $(call GIT_CMD,$(1)) add -A - $(call GIT_CMD,$(1)) commit -m "init" - $(call GIT_CMD,$(1)) tag dist -endef - -define IMPORT_PATCH - cd '$(call GIT_DIR,$(1))' \ - && cat '$(2)' \ - | $(SED) '/^From/,$$ !d' \ - | $(SED) s/'^From: MXE'/"From: fix@me"/'g;' \ - | $(call GIT_CMD,$(1)) am --keep-cr ; -endef - -define EXPORT_PATCH - cd '$(call GIT_DIR,$(1))' \ - && ( \ - echo 'This file is part of MXE. See LICENSE.md for licensing information.'; \ - echo ''; \ - echo 'Contains ad hoc patches for cross building.'; \ - echo ''; \ - $(call GIT_CMD,$(1)) format-patch \ - --numbered \ - -p \ - --no-signature \ - --stdout \ - --text \ - -M9 \ - dist..HEAD \ - | $(SED) 's/^From [0-9a-f]\{40\} /From 0000000000000000000000000000000000000000 /' \ - | $(SED) 's/^index .......\.\......../index 1111111..2222222/' \ - ) > '$(PATCH_BY_NAME)' -endef - -_init-git-%: TIMESTAMP = patch -_init-git-%: download-only-% - $(if $(call set_is_member,$*,$(PKGS)), \ - $(if $(wildcard $(call GIT_DIR,$*)), \ - $(error $(call GIT_DIR,$*) already exists), \ - $(call INIT_GIT,$*)), \ - $(error Package $* not found)) - -_import-patch-%: - $(if $(call set_is_member,$*,$(PKGS)), \ - $(if $(wildcard $(call GIT_DIR,$*)), \ - $(call IMPORT_PATCH,$*,$(call PATCH_BY_NAME,$*,$(PATCH_NAME))), \ - $(error $(call GIT_DIR,$*) does not exist)), \ - $(error Package $* not found)) - -_import-all-patches-%: - $(if $(call set_is_member,$*,$(PKGS)), \ - $(if $(wildcard $(call GIT_DIR,$*)), \ - $(foreach PKG_PATCH,$(call PKG_PATCHES,$*), \ - $(call IMPORT_PATCH,$*,$(PKG_PATCH))), \ - $(error $(call GIT_DIR,$*) does not exist)), \ - $(error Package $* not found)) - -_export-patch-%: - $(if $(call set_is_member,$*,$(PKGS)), \ - $(if $(wildcard $(call GIT_DIR,$*)), \ - $(call EXPORT_PATCH,$*,$(PATCH_NAME)), \ - $(error $(call GIT_DIR,$*) does not exist)), \ - $(error Package $* not found)) - -# use .SECONDARY: when refreshing all patches if you don't -# want to unpack everything every time -#.SECONDARY: -init-git-%: $(PREFIX)/installed/patch/init-git-% ; -import-patch-%: $(PREFIX)/installed/patch/import-patch-% ; -import-all-patches-%: $(PREFIX)/installed/patch/import-all-patches-% ; -export-patch-%: $(PREFIX)/installed/patch/export-patch-% ; - -refresh-patch-%: $(PREFIX)/installed/patch/refresh-patch-% ; -$(PREFIX)/installed/patch/refresh-patch-%: - @rm -rf $(PWD)/tmp-patch/$* - @$(MAKE) -f '$(MAKEFILE)' init-git-$* GITS_DIR=$(PWD)/tmp-patch/$* - @$(MAKE) -f '$(MAKEFILE)' import-patch-$* GITS_DIR=$(PWD)/tmp-patch/$* - @$(MAKE) -f '$(MAKEFILE)' export-patch-$* GITS_DIR=$(PWD)/tmp-patch/$* - @# darwin sometimes chokes deleting large git repos - @rm -rf $(PWD)/tmp-patch/$* || sleep 5; rm -rf $(PWD)/tmp-patch/$* - +@mkdir -p '$(dir $@)' - @touch '$@' - -$(PREFIX)/installed/patch/%: - @echo '[$*]' - @[ -d '$(LOG_DIR)/patch' ] || mkdir -p '$(LOG_DIR)/patch' - @(time $(MAKE) -f '$(MAKEFILE)' _$*) &> '$(LOG_DIR)/patch/$*' - +@mkdir -p '$(dir $@)' - @touch '$@' - - -PATCH_FORMAT_PATCHES := $(shell find $(MXE_PLUGIN_DIRS) plugins -name "*-$(PATCH_NAME).patch") -PATCH_FORMAT_PKGS := $(sort $(subst -$(PATCH_NAME),,$(basename $(notdir $(PATCH_FORMAT_PATCHES))))) -PATCH_FORMAT_DIRS := $(sort $(basename $(dir $(PATCH_FORMAT_PATCHES)))) - -.PHONY: refresh-patches -refresh-patches: - @$(MAKE) -f '$(MAKEFILE)' -j '$(JOBS)'\ - $(addprefix refresh-patch-,$(PATCH_FORMAT_PKGS)) \ - MXE_PLUGIN_DIRS='$(PATCH_FORMAT_DIRS)' diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index d1c3c7f..90949c6 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -1057,14 +1057,16 @@ local function makeMxeSourcePackage() -- dependencies local deps = {} local files = { + 'docs', + 'ext', 'LICENSE.md', 'Makefile', - 'patch.mk', + 'mxe.github.mk', + 'mxe.patch.mk', + 'mxe.updates.mk', + 'plugins', 'README.md', - 'docs', - 'ext', 'src', - 'plugins', 'tools', } local d1 = "MXE source" diff --git a/updates.mk b/updates.mk deleted file mode 100644 index 9a4455d..0000000 --- a/updates.mk +++ /dev/null @@ -1,79 +0,0 @@ -# This file is part of MXE. See LICENSE.md for licensing information. - -# Generic updater -# $(call GET_LATEST_VERSION, base url[, prefix, ext, filter, separator]) -# base url : required page returning list of versions -# e.g https://ftp.gnu.org/gnu/libfoo -# prefix : segment before version -# defaults to lastword of url with dash i.e. `libfoo-` -# ext : segment ending version - default `\.tar` -# filter : `grep -i` filter-out pattern - default alpha\|beta\|rc -# separator: transform char to `.` - typically `_` -# -# test changes with: -# make check-get-latest-version -# -# and update tools/skeleton.py with usage notes - -define GET_LATEST_VERSION - $(WGET) -q -O- '$(strip $(1))' | \ - $(SED) -n 's,.* $(2) ignoring)), - $(info NEW $(1) $($(1)_VERSION) --> $(2)) - $(if $(findstring undefined, $(origin UPDATE_DRYRUN)), - $(SED) -i 's/^\([^ ]*_VERSION *:=\).*/\1 $(2)/' '$($(1)_MAKEFILE)' - $(MAKE) -f '$(MAKEFILE)' 'update-checksum-$(1)' \ - || { $(SED) -i 's/^\([^ ]*_VERSION *:=\).*/\1 $($(1)_VERSION)/' '$($(1)_MAKEFILE)'; \ - exit 1; }))), - $(info Unable to update version number of package $(1) \ - $(newline)$(newline)$($(1)_UPDATE)$(newline))) - -endef -update: - $(foreach PKG,$(PKGS),\ - $(and $($(PKG)_UPDATE),$(call UPDATE,$(PKG),$(shell $($(PKG)_UPDATE))))) - -update-package-%: - $(if $(call set_is_member,$*,$(PKGS)), \ - $(and $($*_UPDATE),$(call UPDATE,$*,$(shell $($*_UPDATE)))), \ - $(error Package $* not found)) - @echo -n - -check-update-package-%: UPDATE_DRYRUN = true -check-update-package-%: update-package-% ; - -update-checksum-%: MXE_NO_BACKUP_DL = true -update-checksum-%: SKIP_CHECHSUM = true -update-checksum-%: - $(if $(call set_is_member,$*,$(PKGS)), \ - $(call DOWNLOAD_PKG_ARCHIVE,$*) && \ - $(SED) -i 's/^\([^ ]*_CHECKSUM *:=\).*/\1 '"`$(call PKG_CHECKSUM,$*)`"'/' '$($*_MAKEFILE)', \ - $(error Package $* not found)) -- cgit v0.12