Merge pull request #1712 from tonytheodore/gh-std

Standardise GitHub downloads and updates

10 files changed, 197 insertions, 57 deletions

diff --git a/Makefile b/Makefile
index f0e5ba3..c9e4440 100644
--- a/Makefile
+++ b/Makefile
@@ -69,6 +69,7 @@ define newline
 endef
 
 \n    := $(newline)
+comma := ,
 null  :=
 space := $(null) $(null)
 repeat = $(subst x,$(1),$(subst $(space),,$(call int_encode,$(2))))
@@ -158,6 +159,7 @@ MAKE_SHARED_FROM_STATIC = \
 	--libdir '$(PREFIX)/$(TARGET)/lib' \
 	--bindir '$(PREFIX)/$(TARGET)/bin'
 
+# MXE_GET_GITHUB functions can be removed once all packages use GH_CONF
 define MXE_GET_GITHUB_SHA
     $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/heads/$(strip $(2))' \
     | $(SED) -n 's#.*"sha": "\([^"]\{10\}\).*#\1#p' \
@@ -176,6 +178,9 @@ define MXE_GET_GITHUB_TAGS
     | tail -1
 endef
 
+# include github related functions
+include $(PWD)/github.mk
+
 # shared lib preload to disable networking, enable faketime etc
 PRELOAD_VARS := LD_PRELOAD DYLD_FORCE_FLAT_NAMESPACE DYLD_INSERT_LIBRARIES
 
@@ -467,6 +472,9 @@ endef
 $(foreach TARGET,$(MXE_TARGETS),$(call TARGET_RULE,$(TARGET)))
 
 define PKG_RULE
+# configure GitHub metadata if GH_CONF is set
+$(if $($(PKG)_GH_CONF),$(eval $(MXE_SETUP_GITHUB)))
+
 .PHONY: download-$(1)
 download-$(1): $(addprefix download-,$($(1)_DEPS)) download-only-$(1)
 
@@ -720,12 +728,16 @@ clean-pkg:
 clean-junk: clean-pkg
 	rm -rf $(LOG_DIR) $(call TMP_DIR,*)
 
+COMPARE_VERSIONS = $(strip \
+    $(if $($(1)_BRANCH),$(call seq,$($(1)_VERSION),$(2)),\
+    $(filter $(2),$(shell printf '$($(1)_VERSION)\n$(2)' | $(SORT) -V | head -1))))
+
 .PHONY: update
 define UPDATE
     $(if $(2),
         $(if $(filter $($(1)_IGNORE),$(2)),
             $(info IGNORED  $(1)  $(2)),
-            $(if $(filter $(2),$(shell printf '$($(1)_VERSION)\n$(2)' | $(SORT) -V | head -1)),
+            $(if $(COMPARE_VERSIONS),
                 $(if $(filter $(2),$($(1)_VERSION)),
                     $(info .        $(1)  $(2)),
                     $(info OLD      $(1)  $($(1)_VERSION) --> $(2) ignoring)),
@@ -747,6 +759,10 @@ update-package-%:
 	$(if $(call set_is_member,$*,$(PKGS)), \
 	    $(and $($*_UPDATE),$(call UPDATE,$*,$(shell $($*_UPDATE)))), \
 	    $(error Package $* not found))
+	    @echo -n
+
+check-update-package-%: UPDATE_DRYRUN = true
+check-update-package-%: update-package-% ;
 
 update-checksum-%: MXE_NO_BACKUP_DL = true
 update-checksum-%:
diff --git a/docs/index.html b/docs/index.html
index b93992f..293579b 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -1248,6 +1248,35 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
         </p>
 
         <p>
+        GitHub hosted projects can automatically configure updates,
+        urls, file names etc. by setting <code>$(PKG)_GH_CONF</code>
+        instead of <code>$(PKG)_FILE, $(PKG)_SUBDIR, $(PKG)_URL, and
+        $(PKG)_UPDATE</code> sections.
+        </p>
+        <p>
+        To track normal release tags set:
+        <pre>$(PKG)_GH_CONF  := owner/repo[, tag prefix, tag suffix, tag filter-out, version separator]</pre>
+        To track branches, set:
+        <pre>$(PKG)_GH_CONF  := owner/repo/branch</pre>
+        See the following packages for examples:
+        <ul>
+            <li>
+            <a href="https://github.com/mxe/mxe/blob/master/src/vmime.mk">vmime.mk</a>
+            for branch tracking
+            </li>
+            <li>
+            <a href="https://github.com/mxe/mxe/blob/master/src/libevent.mk">libevent.mk</a>
+            for tag tracking
+            </li>
+            <li>
+            <a href="https://github.com/mxe/mxe/blob/master/src/libffi.mk">libffi.mk</a>
+            for externally hosted tarballs with generated sources not
+            present in source tree
+            </li>
+        </ul>
+        </p>
+
+        <p>
         The <a href="gmsl.html">GNU Make Standard Library</a> is also
         available (though it should be unnecessary for most packages).
         </p>
diff --git a/github.mk b/github.mk
new file mode 100644
index 0000000..cf6edbe
--- /dev/null
+++ b/github.mk
@@ -0,0 +1,114 @@
+# This file is part of MXE. See LICENSE.md for licensing information.
+
+# Standardise GitHub downloads and updates
+# Download API has two forms:
+#   Archive:
+#     url = <owner>/<repo>/archive/<ref>.tar.gz
+#     dir = <repo>-<ref>
+#
+#   Tarball:
+#     url = <owner>/<repo>/tarball/<ref>/output-file.tar.gz
+#     dir = <owner>-<repo>-<short sha>
+#
+## also third api - `releases` see libass.mk
+#
+# Filename doesn't matter as we stream the url to a name of our choosing.
+#
+# The archive API could be used for all packages, however, if the reference
+# is any sort of SHA, the full SHA is used for the directory. We could `cd`
+# into it without knowing the SHA beforehand, but the directory length would
+# be comical in logs etc.
+#
+# The tarball API accepts references to commits or tags, always using the
+# short SHA as the directory. In this case, tag tracking packages would have
+# to store the SHA (see #1002). However, this only works for lightweight
+# tags, not annotated tags that most projects use for releases.
+#
+# In summary, we have to use both.
+
+# The tarball API determines the short SHA length used in the directory name.
+# Chances of a collision on a "given" commit seem to decrease as the chance
+# of "any" collision increases. If that changes in the future, `make update`
+# will fix it.
+# Currently (2015-12) the API sets the short SHA length to:
+GITHUB_SHA_LENGTH := 7
+
+# Packages must set the following metadata:
+#   Track branch - Tarball API
+#     GH_CONF := owner/repo/branch
+#     updates will use the last commit from the specified branch as
+#     a version string and bypass `sort -V`
+#
+#   Track tags - Archive API
+#     GH_CONF := owner/repo, tag prefix, tag suffix, tag filter-out, version separator
+#     updates will construct a version number based on:
+#     <tag prefix><s/<version sep>/./version><tag suffix>
+
+# common tag filtering is applied with `grep -v`:
+GITHUB_TAG_FILTER := alpha\|beta\|rc
+
+# More complex filters can fall back to `MXE_GET_GH_TAGS` which returns
+# a list for post-processing.
+
+# ...and finally, auto-configure packages based on above metadata:
+#   - `eval` these snippets during PKG_RULE loop (where PKG is in scope).
+#   - `call` or `eval` from package makefiles requires complex quoting
+#     and looks out of place.
+#   - don't redefine manually set standard variables (FILE, SUBDIR, URL, UPDATE)
+
+GH_REPO        = $(subst $(space),/,$(wordlist 1,2,$(subst /,$(space),$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))
+GH_BRANCH      = $(word 3,$(subst /,$(space),$(word 1,$(subst $(comma),$(space),$($(PKG)_GH_CONF)))))
+GH_TAG_VARS    = $(call rest,$(subst $(comma),$(space)$(__gmsl_aa_magic),$(subst $(space),,$($(PKG)_GH_CONF))))
+GH_TAG_PREFIX  = $(subst $(__gmsl_aa_magic),,$(word 1,$(GH_TAG_VARS)))
+GH_TAG_SUFFIX  = $(subst $(__gmsl_aa_magic),,$(word 2,$(GH_TAG_VARS)))
+GH_TAG_FILTER  = $(subst $(__gmsl_aa_magic),,$(word 3,$(GH_TAG_VARS)))
+GH_VERSION_SEP = $(subst $(__gmsl_aa_magic),,$(word 4,$(GH_TAG_VARS)))
+
+define MXE_SETUP_GITHUB
+    $(PKG)_GH_REPO     := $(GH_REPO)
+    $(PKG)_BRANCH      := $(GH_BRANCH)
+    $(PKG)_TAG_VARS    := $(GH_TAG_VARS)
+    $(PKG)_TAG_PREFIX  := $(GH_TAG_PREFIX)
+    $(PKG)_TAG_SUFFIX  := $(GH_TAG_SUFFIX)
+    $(PKG)_TAG_FILTER  := $(GH_TAG_FILTER)
+    $(PKG)_VERSION_SEP := $(or $(GH_VERSION_SEP),.)
+    $(PKG)_FILE        := $(or $($(PKG)_FILE),$(PKG)-$$($$(PKG)_TAG_PREFIX)$($(PKG)_VERSION)$$($$(PKG)_TAG_SUFFIX).tar.gz)
+    $(if $(and $(GH_BRANCH),$(GH_TAG_VARS)),\
+        $(error $(newline) $(PKG) specifies both branch and tag variables $(newline)))
+    $(if $(GH_BRANCH),$(value MXE_SETUP_GITHUB_BRANCH),$(value MXE_SETUP_GITHUB_TAG))
+endef
+
+define MXE_SETUP_GITHUB_BRANCH
+    $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$(subst /,-,$($(PKG)_GH_REPO))-$($(PKG)_VERSION))
+    $(PKG)_URL    := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_REPO)/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE))
+    $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_SHA,$($(PKG)_GH_REPO),$($(PKG)_BRANCH)))
+endef
+
+define MXE_SETUP_GITHUB_TAG
+    $(PKG)_SUBDIR := $(or $($(PKG)_SUBDIR),$(PKG)-$($(PKG)_TAG_PREFIX)$(subst .,$($(PKG)_VERSION_SEP),$($(PKG)_VERSION))$($(PKG)_TAG_SUFFIX))
+    $(PKG)_URL    := $(or $($(PKG)_URL),https://github.com/$($(PKG)_GH_REPO)/archive/$(subst $(PKG)-,,$($(PKG)_SUBDIR)).tar.gz)
+    $(PKG)_UPDATE := $(or $($(PKG)_UPDATE),$(call MXE_GET_GH_TAG,$($(PKG)_GH_REPO),$($(PKG)_TAG_PREFIX),$($(PKG)_TAG_SUFFIX),$(or $($(PKG)_TAG_FILTER),$(GITHUB_TAG_FILTER)),$($(PKG)_VERSION_SEP)))
+endef
+
+# called with owner/repo,branch
+define MXE_GET_GH_SHA
+    $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/heads/$(strip $(2))' \
+    | $(SED) -n 's#.*"sha": "\([^"]\{$(GITHUB_SHA_LENGTH)\}\).*#\1#p' \
+    | head -1
+endef
+
+# called with owner/repo
+define MXE_GET_GH_TAGS
+    $(WGET) -q -O- 'https://api.github.com/repos/$(strip $(1))/git/refs/tags/' \
+    | $(SED) -n 's#.*"ref": "refs/tags/\([^"]*\).*#\1#p'
+endef
+
+# called with owner/repo, tag prefix, tag suffix, filter-out, version sep
+define MXE_GET_GH_TAG
+    $(MXE_GET_GH_TAGS) \
+    | $(if $(4),grep -v '$(strip $(4))') \
+    | $(SED) -n 's,^$(strip $(2))\([^"]*\)$(strip $(3))$$,\1,p' \
+    | tr '$(strip $(5))' '.' \
+    | $(SORT) -V
+    | tail -1
+endef
diff --git a/src/libevent.mk b/src/libevent.mk
index ea9f9ef..afeb739 100644
--- a/src/libevent.mk
+++ b/src/libevent.mk
@@ -5,18 +5,9 @@ $(PKG)_WEBSITE  := http://libevent.org/
 $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := 2.0.21
 $(PKG)_CHECKSUM := 872b7cdc199ead2edd9f0d1e93b4d900e67d892c014545bd3314b3ae49505eff
-$(PKG)_SUBDIR   := libevent-release-$($(PKG)_VERSION)-stable
-$(PKG)_FILE     := release-$($(PKG)_VERSION)-stable.tar.gz
-$(PKG)_URL      := https://github.com/$(PKG)/$(PKG)/archive/$($(PKG)_FILE)
+$(PKG)_GH_CONF  := libevent/libevent, release-, -stable
 $(PKG)_DEPS     := gcc openssl
 
-define $(PKG)_UPDATE
-    $(WGET) -q -O- 'http://libevent.org/' | \
-    grep 'libevent-' | \
-    $(SED) -n 's,.*libevent-\([0-9][^>]*\)-stable\.tar.*,\1,p' | \
-    head -1
-endef
-
 define $(PKG)_BUILD
     cd '$(1)' && ./autogen.sh && OPENSSL_LIBADD=-lz ./configure \
         $(MXE_CONFIGURE_OPTS)
diff --git a/src/libffi.mk b/src/libffi.mk
index 2db18ef..27bbbd5 100644
--- a/src/libffi.mk
+++ b/src/libffi.mk
@@ -5,19 +5,13 @@ $(PKG)_WEBSITE  := https://sourceware.org/libffi/
 $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := 3.2.1
 $(PKG)_CHECKSUM := d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37
+$(PKG)_GH_CONF  := atgreen/libffi, v
 $(PKG)_SUBDIR   := $(PKG)-$($(PKG)_VERSION)
 $(PKG)_FILE     := $(PKG)-$($(PKG)_VERSION).tar.gz
 $(PKG)_URL      := https://www.mirrorservice.org/sites/sourceware.org/pub/$(PKG)/$($(PKG)_FILE)
 $(PKG)_URL_2    := ftp://sourceware.org/pub/$(PKG)/$($(PKG)_FILE)
 $(PKG)_DEPS     := gcc
 
-define $(PKG)_UPDATE
-    $(WGET) -q -O- 'https://github.com/atgreen/libffi/tags' | \
-    grep '<a href="/atgreen/libffi/archive/' | \
-    $(SED) -n 's,.*href="/atgreen/libffi/archive/v\([0-9][^"]*\)\.tar.*,\1,p' | \
-    head -1
-endef
-
 define $(PKG)_BUILD
     cd '$(1)' && ./configure \
         $(MXE_CONFIGURE_OPTS)
diff --git a/src/libsoup.mk b/src/libsoup.mk
index efb35dc..2873f68 100644
--- a/src/libsoup.mk
+++ b/src/libsoup.mk
@@ -7,15 +7,9 @@ $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := 2.57.1
 $(PKG)_APIVER   := 2.4
 $(PKG)_CHECKSUM := 675c3bc11c2a6347625ca5215720d41c84fd8e9498dd664cda8a635fd5105a26
-$(PKG)_SUBDIR   := $(PKG)-$($(PKG)_VERSION)
-$(PKG)_FILE     := $(PKG)-$($(PKG)_VERSION).tar.gz
-$(PKG)_URL      := https://github.com/GNOME/$(PKG)/archive/$($(PKG)_VERSION).tar.gz
+$(PKG)_GH_CONF  := GNOME/libsoup,,,pre\|SOUP\|base
 $(PKG)_DEPS     := gcc glib libxml2 sqlite
 
-define $(PKG)_UPDATE
-    $(call MXE_GET_GITHUB_TAGS, GNOME/libsoup)
-endef
-
 define $(PKG)_BUILD
     cd '$(SOURCE_DIR)' && \
         NOCONFIGURE=1 \
diff --git a/src/qtserialport_qt4.mk b/src/qtserialport_qt4.mk
index dbe37bf..13dd068 100644
--- a/src/qtserialport_qt4.mk
+++ b/src/qtserialport_qt4.mk
@@ -4,18 +4,11 @@ PKG             := qtserialport_qt4
 $(PKG)_WEBSITE  := https://www.qt.io/
 $(PKG)_DESCR    := Qt
 $(PKG)_IGNORE   :=
-$(PKG)_VERSION  := 5c3b6cc770
+$(PKG)_VERSION  := 5c3b6cc
 $(PKG)_CHECKSUM := d49c1cd4bb47706561f52c07d6075bb9931700d3bcae656ef3b6d3db3eb014ab
-$(PKG)_GH_USER  := qt
-$(PKG)_GH_REPO  := qtserialport
-$(PKG)_GH_TREE  := qt4-dev
-$(PKG)_SUBDIR   := $($(PKG)_GH_USER)-$($(PKG)_GH_REPO)-$(call substr,$($(PKG)_VERSION),1,7)
-$(PKG)_FILE     := $(PKG)-$($(PKG)_VERSION).tar.gz
-$(PKG)_URL      := https://github.com/$($(PKG)_GH_USER)/$($(PKG)_GH_REPO)/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE)
+$(PKG)_GH_CONF  := qtproject/qtserialport/qt4-dev
 $(PKG)_DEPS     := gcc qt
 
-$(PKG)_UPDATE   := $(call MXE_GET_GITHUB_SHA, $($(PKG)_GH_USER)/$($(PKG)_GH_REPO), $($(PKG)_GH_TREE))
-
 define $(PKG)_BUILD
     cd '$(1)' && '$(PREFIX)/$(TARGET)/qt/bin/qmake'
     $(MAKE) -C '$(1)' -j '$(JOBS)'
diff --git a/src/vigra.mk b/src/vigra.mk
index e8c1b08..32c6b4b 100644
--- a/src/vigra.mk
+++ b/src/vigra.mk
@@ -5,20 +5,9 @@ $(PKG)_WEBSITE  := https://ukoethe.github.io/vigra/
 $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := 1.9.0
 $(PKG)_CHECKSUM := dc041f7ccf838d4321e9bcf522fece1758768dd7a3f8350d1e83e2b8e6daf1e6
-$(PKG)_SUBDIR   := vigra-Version-$(subst .,-,$($(PKG)_VERSION))
-$(PKG)_FILE     := vigra-$($(PKG)_VERSION).tar.gz
-$(PKG)_URL      := https://github.com/ukoethe/vigra/archive/Version-$(subst .,-,$($(PKG)_VERSION)).tar.gz
+$(PKG)_GH_CONF  := ukoethe/vigra, Version-,,,-
 $(PKG)_DEPS     := gcc jpeg libpng openexr tiff
 
-define $(PKG)_UPDATE
-    $(WGET) -q -O- "https://api.github.com/repos/ukoethe/vigra/releases" | \
-    grep 'tag_name' | \
-    $(SED) -n 's,.*tag_name": "Version-\([0-9][^>]*\)".*,\1,p' | \
-    tr '-' '.' | \
-    $(SORT) -Vr | \
-    head -1
-endef
-
 define $(PKG)_BUILD
     # Make sure the package gets built statically
     # NB: we're not actually building vigranumpy, but preparing it in case we ever will won't hurt
diff --git a/src/vmime.mk b/src/vmime.mk
index 2fd745a..8aa0830 100644
--- a/src/vmime.mk
+++ b/src/vmime.mk
@@ -6,13 +6,9 @@ $(PKG)_DESCR    := VMime
 $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := 3cfcdca
 $(PKG)_CHECKSUM := 45d2c1dcf3945939ec921c9292769faf4bf38a8fbe9c276b3bc387b4625a1841
-$(PKG)_SUBDIR   := kisli-vmime-$($(PKG)_VERSION)
-$(PKG)_FILE     := $(PKG)-$($(PKG)_VERSION).tar.gz
-$(PKG)_URL      := https://github.com/kisli/vmime/tarball/$($(PKG)_VERSION)/$($(PKG)_FILE)
+$(PKG)_GH_CONF  := kisli/vmime/master
 $(PKG)_DEPS     := gcc gnutls libgsasl libiconv pthreads zlib
 
-$(PKG)_UPDATE    = $(call MXE_GET_GITHUB_SHA, kisli/vmime, master) | $(SED) 's/^\(.......\).*/\1/;'
-
 define $(PKG)_BUILD
     # The following hint is probably needed for ICU:
     # -DICU_LIBRARIES="`'$(TARGET)-pkg-config' --libs-only-l icu-i18n`"
diff --git a/tools/skeleton.py b/tools/skeleton.py
index 441243c..c66f2df 100755
--- a/tools/skeleton.py
+++ b/tools/skeleton.py
@@ -27,19 +27,29 @@ $(PKG)_DESCR    := %(description)s
 $(PKG)_IGNORE   :=
 $(PKG)_VERSION  := %(version)s
 $(PKG)_CHECKSUM := %(checksum)s
+%(file_specs)s
+$(PKG)_DEPS     := gcc
+%(update)s
+define $(PKG)_BUILD
+    %(build)s
+endef
+'''
+
+GH_CONF=r'''
+$(PKG)_GH_CONF  := %(gh_conf)s
+'''
+
+FILE_SPECS=r'''
 $(PKG)_SUBDIR   := %(subdir_template)s
 $(PKG)_FILE     := %(filename_template)s
 $(PKG)_URL      := %(file_url_template)s
-$(PKG)_DEPS     := gcc
+'''
 
+UPDATE = r'''
 define $(PKG)_UPDATE
     echo 'TODO: write update script for %(name)s.' >&2;
     echo $(%(name)s_VERSION)
 endef
-
-define $(PKG)_BUILD
-    %(build)s
-endef
 '''
 
 CMAKE_BUILD = r'''
@@ -92,6 +102,10 @@ PC_AND_TEST = r'''
 def get_filename(file_url):
     return file_url.rsplit('/', 1)[1]
 
+def deduce_gh_conf(file_url):
+    if deduce_website(file_url) == 'github.com':
+        return '/'.join(file_url.split('://', 1)[1].split('/')[1:3])
+
 def deduce_version(file_url):
     filename = get_filename(file_url)
     return re.search(r'\d[\d.-_]+\d|\d', filename).group()
@@ -137,7 +151,7 @@ def make_skeleton(
     website,
     builder,
 ):
-    mk_filename = 'src/%s.mk' % name
+    mk_filename = 'src/%s.mk' % name.lower()
     if os.path.isfile(mk_filename):
         raise Exception('File %s exists!' % mk_filename)
     if description is None:
@@ -146,6 +160,13 @@ def make_skeleton(
         version = deduce_version(file_url)
     if website is None:
         website = deduce_website(file_url)
+    gh_conf = deduce_gh_conf(file_url)
+    if gh_conf is None:
+        file_specs = FILE_SPECS
+        update = UPDATE
+    else:
+        file_specs = GH_CONF
+        update = ''
     with tempfile.NamedTemporaryFile() as pkg_file:
         download_file(pkg_file.name, file_url)
         checksum = make_checksum(pkg_file.name)
@@ -165,11 +186,14 @@ def make_skeleton(
             'libname': libname,
             'website': website,
             'file_url_template': file_url_template,
+            'gh_conf': gh_conf,
             'checksum': checksum,
             'version': version,
             'subdir_template': subdir_template,
             'filename_template': filename_template,
         }
+        options['file_specs'] = file_specs.strip() % options
+        options['update'] = update % options
         options['build'] = make_build(options, builder)
         mk.write(MK_TEMPLATE.lstrip() % options)