update build-pkg.lua for toolchain re-org

1. Remove common packages.

  Common packages served a replacement for native target.
  Now we have native target and don't need common packages.
  Existing common files (ncurses) have to be fixed.

2. Package -> Item.

  Item means a string "target~package".
  All functions which used packages now use items.

3. One build list instead of 4 build lists.

  All items are sorted and built together without separation
  by target.

4. No module-global variable "target".

  All functions using target now get "item" and target is
  extracted from item. All remaining module-global variables
  don't change (are constants) or are created in the bottom
  of the module.

5. MXE_MAX_PACKAGES -> MXE_MAX_ITEMS

see #925
see #919

1 files changed, 103 insertions, 178 deletions

diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua
index 0b02dfd..bb0492d 100755
--- a/tools/build-pkg.lua
+++ b/tools/build-pkg.lua
@@ -22,7 +22,7 @@ set environment variable MXE_NO_DEBS to 1
 In this case fakeroot and dpkg-deb are not needed.
 
 To limit number of packages being built to x,
-set environment variable MXE_MAX_PACKAGES to x,
+set environment variable MXE_MAX_ITEMS to x,
 
 The following error:
 > fakeroot, while creating message channels: Invalid argument
@@ -32,7 +32,7 @@ can be caused by leaked ipc resources originating in fakeroot.
 How to remove them: http://stackoverflow.com/a/4262545
 ]]
 
-local max_packages = tonumber(os.getenv('MXE_MAX_PACKAGES'))
+local max_items = tonumber(os.getenv('MXE_MAX_ITEMS'))
 local no_debs = os.getenv('MXE_NO_DEBS')
 
 local ARCH = 'amd64'
@@ -48,14 +48,6 @@ local BLACKLIST = {
     '^usr/[^/]+/share/info/',
 }
 
-local COMMON_FILES = {
-    ['ncurses'] = {
-        '^usr/lib/pkgconfig/',
-    },
-}
-
-local ARCH_FOR_COMMON = 'i686-w64-mingw32.static'
-
 local TARGETS = {
     'i686-w64-mingw32.static',
     'x86_64-w64-mingw32.static',
@@ -63,10 +55,8 @@ local TARGETS = {
     'x86_64-w64-mingw32.shared',
 }
 
-local target -- used by many functions
-
 local function log(fmt, ...)
-    print('[build-pkg]', target, fmt:format(...))
+    print('[build-pkg]', fmt:format(...))
 end
 
 -- based on http://lua-users.org/wiki/SplitJoin
@@ -109,6 +99,15 @@ local function isInArray(element, array)
     return false
 end
 
+local function sliceArray(list, nelements)
+    nelements = nelements or #list
+    local new_list = {}
+    for i = 1, nelements do
+        new_list[i] = list[i]
+    end
+    return new_list
+end
+
 local function shell(cmd)
     local f = io.popen(cmd, 'r')
     local text = f:read('*all')
@@ -151,34 +150,22 @@ local function fileExists(name)
     end
 end
 
--- return several tables describing packages
--- * list of packages
--- * map from package to list of deps
--- * map from package to version of package
-local function getPkgs()
-    -- create file deps.mk showing deps
-    -- (make show-upstream-deps-% does not present in
-    -- stable MXE)
-    local deps_mk_content = [[
-include Makefile
-NOTHING:=
-SPACE:=$(NOTHING) $(NOTHING)
-NAME_WITH_UNDERSCORES:=$(subst $(SPACE),_,$(NAME))
-print-deps:
-	@$(foreach pkg,$(PKGS),echo \
-		for-build-pkg $(pkg) \
-		$(subst $(SPACE),-,$($(pkg)_VERSION)) \
-		$($(pkg)_DEPS) \
-		$(if $(call set_is_not_member,$(pkg),$(MXE_CONF_PKGS)), \
-		$(MXE_CONF_PKGS));)]]
-    local deps_mk_file = io.open('deps.mk', 'w')
-    deps_mk_file:write(deps_mk_content)
-    deps_mk_file:close()
-    local pkgs = {}
-    local pkg2deps = {}
-    local pkg2ver = {}
-    local cmd = '%s -f deps.mk print-deps MXE_TARGETS=%s'
-    cmd = cmd:format(tool 'make', target)
+-- return target and package from item name
+local function parseItem(item)
+    return item:match("([^~]+)~([^~]+)")
+end
+
+-- return several tables describing packages for all targets
+-- * list of items
+-- * map from item to list of deps (which are also items)
+-- * map from item to version
+-- Item is a string like "target~pkg"
+local function getItems()
+    local items = {}
+    local item2deps = {}
+    local item2ver = {}
+    local cmd = '%s print-deps-for-build-pkg MXE_TARGETS=%q'
+    cmd = cmd:format(tool 'make', table.concat(TARGETS, ' '))
     local make = io.popen(cmd)
     for line in make:lines() do
         local deps = split(trim(line))
@@ -186,29 +173,28 @@ print-deps:
             -- first value is marker 'for-build-pkg'
             table.remove(deps, 1)
             -- first value is name of package which depends on
-            local pkg = table.remove(deps, 1)
+            local item = table.remove(deps, 1)
             -- second value is version of package
             local ver = table.remove(deps, 1)
-            table.insert(pkgs, pkg)
-            pkg2deps[pkg] = deps
-            pkg2ver[pkg] = ver
+            table.insert(items, item)
+            item2deps[item] = deps
+            item2ver[item] = ver
         end
     end
     make:close()
-    os.remove('deps.mk')
-    return pkgs, pkg2deps, pkg2ver
+    return items, item2deps, item2ver
 end
 
--- return packages ordered in build order
--- this means, if pkg1 depends on pkg2, then
--- pkg2 preceeds pkg1 in the list
-local function sortForBuild(pkgs, pkg2deps)
+-- return items ordered in build order
+-- this means, if item depends on item2, then
+-- item2 preceeds item1 in the list
+local function sortForBuild(items, item2deps)
     -- use sommand tsort
     local tsort_input_fname = os.tmpname()
     local tsort_input = io.open(tsort_input_fname, 'w')
-    for _, pkg1 in ipairs(pkgs) do
-        for _, pkg2 in ipairs(pkg2deps[pkg1]) do
-            tsort_input:write(pkg2 .. ' ' .. pkg1 .. '\n')
+    for _, item1 in ipairs(items) do
+        for _, item2 in ipairs(item2deps[item1]) do
+            tsort_input:write(item2 .. ' ' .. item1 .. '\n')
         end
     end
     tsort_input:close()
@@ -216,8 +202,8 @@ local function sortForBuild(pkgs, pkg2deps)
     local build_list = {}
     local tsort = io.popen('tsort ' .. tsort_input_fname, 'r')
     for line in tsort:lines() do
-        local pkg = trim(line)
-        table.insert(build_list, pkg)
+        local item = trim(line)
+        table.insert(build_list, item)
     end
     tsort:close()
     os.remove(tsort_input_fname)
@@ -287,12 +273,13 @@ local function gitCommit(message)
     os.execute(cmd:format(message))
 end
 
-local function isValidBinary(file)
+local function isValidBinary(target, file)
     local cmd = './usr/bin/%s-objdump -t %s > /dev/null 2>&1'
     return execute(cmd:format(target, file))
 end
 
-local function checkFile(file, pkg)
+local function checkFile(file, item)
+    local target, _ = parseItem(item)
     -- if it is PE32 file, it must have '.exe' in name
     local ext = file:sub(-4):lower()
     local cmd = 'file --dereference --brief %q'
@@ -302,73 +289,66 @@ local function checkFile(file, pkg)
     elseif ext == '.exe' then
         if not file_type:match('PE32') then
             log('File %s (%s) is %q. Remove .exe',
-                file, pkg, file_type)
+                file, item, file_type)
         end
     elseif ext == '.dll' then
         if not file_type:match('PE32.*DLL') then
             log('File %s (%s) is %q. Remove .dll',
-                file, pkg, file_type)
+                file, item, file_type)
         end
     else
         if file_type:match('PE32') then
             log('File %s (%s) is %q. Add exe or dll',
-                file, pkg, file_type)
+                file, item, file_type)
         end
     end
     for _, t in ipairs(TARGETS) do
         if t ~= target and file:match(t) then
             log('File %s (%s): other target %s in name',
-                file, pkg, t)
+                file, item, t)
         end
     end
     if file:match('/lib/.*%.dll$') then
-        log('File %s (%s): DLL in /lib/', file, pkg)
+        log('File %s (%s): DLL in /lib/', file, item)
     end
     if file:match('%.dll$') or file:match('%.a$') then
-        if file:find(target, 1, true) then -- not common
-            if not isValidBinary(file) then
+        if file:find(target, 1, true) then -- cross-compiled
+            if not isValidBinary(target, file) then
                 log('File %s (%s): not recognized library',
-                    file, pkg)
+                    file, item)
             end
         end
     end
 end
 
 -- builds package, returns list of new files
-local function buildPackage(pkg, pkg2deps, file2pkg)
-    local cmd = '%s %s MXE_TARGETS=%s --jobs=1'
-    os.execute(cmd:format(tool 'make', pkg, target))
+local function buildItem(item, item2deps, file2item)
+    local cmd = '%s %s --jobs=1'
+    os.execute(cmd:format(tool 'make', item))
     gitAdd()
     local new_files, changed_files = gitStatus()
-    gitCommit(("Build %s for target %s"):format(pkg, target))
+    gitCommit(("Build %s"):format(item))
     for _, file in ipairs(new_files) do
-        checkFile(file, pkg)
-        file2pkg[file] = {pkg=pkg, target=target}
+        checkFile(file, item)
+        file2item[file] = item
     end
     for _, file in ipairs(changed_files) do
-        checkFile(file, pkg)
+        checkFile(file, item)
         -- add a dependency on a package created this file
-        local creator_pkg = assert(file2pkg[file]).pkg
-        local creator_target = assert(file2pkg[file]).target
-        local level = ''
-        if target == creator_target then
-            if not isInArray(creator_pkg, pkg2deps[pkg]) then
-                table.insert(pkg2deps[pkg], creator_pkg)
-            end
-        else
-            level = 'error'
+        local creator_item = assert(file2item[file])
+        if not isInArray(creator_item, item2deps[item]) then
+            table.insert(item2deps[item], creator_item)
         end
-        log('Package %s changes %s, created by %s (%s) %s',
-            pkg, file, creator_pkg, creator_target, level)
+        log('Item %s changes %s, created by %s',
+            item, file, creator_item)
     end
     return new_files
 end
 
-local function nameToDebian(pkg, t)
-    local name = 'mxe-%s-%s'
-    name = name:format(t or target, pkg)
-    name = name:gsub('_', '-')
-    return name
+local function nameToDebian(item)
+    item = item:gsub('[~_]', '-')
+    local name = 'mxe-%s'
+    return name:format(item)
 end
 
 local function protectVersion(ver)
@@ -381,7 +361,8 @@ local function protectVersion(ver)
     end
 end
 
-local function listFile(pkg)
+local function listFile(item)
+    local target, pkg = parseItem(item)
     return ('%s-%s.list'):format(target, pkg)
 end
 
@@ -401,8 +382,9 @@ Description: MXE package %s for %s
  This package contains the files for MXE package %s.
 ]]
 
-local function makeDeb(pkg, list_path, deps, ver, add_common)
-    local deb_pkg = nameToDebian(pkg)
+local function makeDeb(item, list_path, deps, ver)
+    local target, pkg = parseItem(item)
+    local deb_pkg = nameToDebian(item)
     local dirname = ('%s_%s'):format(deb_pkg,
         protectVersion(ver))
     -- make .tar.xz file
@@ -423,9 +405,6 @@ local function makeDeb(pkg, list_path, deps, ver, add_common)
     for _, dep in ipairs(deps) do
         table.insert(deb_deps, nameToDebian(dep))
     end
-    if add_common then
-        table.insert(deb_deps, nameToDebian(pkg, 'common'))
-    end
     local deb_deps_str = table.concat(deb_deps, ', ')
     -- make DEBIAN/control file
     os.execute(('mkdir -p %s/DEBIAN'):format(dirname))
@@ -443,14 +422,6 @@ local function makeDeb(pkg, list_path, deps, ver, add_common)
     os.execute(('rm -fr %s deb.fakeroot'):format(dirname))
 end
 
-local function readFileList(list_file)
-    local list = {}
-    for installed_file in io.lines(list_file) do
-        table.insert(list, installed_file)
-    end
-    return list
-end
-
 local function saveFileList(list_file, list)
     local file = io.open(list_file, 'w')
     for _, installed_file in ipairs(list) do
@@ -459,7 +430,8 @@ local function saveFileList(list_file, list)
     file:close()
 end
 
-local function isBuilt(pkg, files)
+local function isBuilt(item, files)
+    local target, pkg = parseItem(item)
     local INSTALLED = 'usr/%s/installed/%s'
     local installed = INSTALLED:format(target, pkg)
     for _, file in ipairs(files) do
@@ -470,94 +442,46 @@ local function isBuilt(pkg, files)
     return false
 end
 
--- build all packages, save filelist to file #pkg.list
-local function buildPackages(pkgs, pkg2deps, file2pkg)
+-- build all packages, save filelist to list file
+local function buildPackages(items, item2deps)
     local broken = {}
     local unbroken = {}
-    local function brokenDep(pkg)
-        for _, dep in ipairs(pkg2deps[pkg]) do
+    local file2item = {}
+    local function brokenDep(item)
+        for _, dep in ipairs(item2deps[item]) do
             if broken[dep] then
                 return dep
             end
         end
         return false
     end
-    for _, pkg in ipairs(pkgs) do
-        if not brokenDep(pkg) then
-            local files = buildPackage(pkg, pkg2deps, file2pkg)
-            if isBuilt(pkg, files) then
-                saveFileList(listFile(pkg), files)
-                table.insert(unbroken, pkg)
+    for _, item in ipairs(items) do
+        if not brokenDep(item) then
+            local files = buildItem(item, item2deps, file2item)
+            if isBuilt(item, files) then
+                saveFileList(listFile(item), files)
+                table.insert(unbroken, item)
             else
                 -- broken package
-                broken[pkg] = true
-                log('The package is broken: %s', pkg)
+                broken[item] = true
+                log('Item is broken: %s', item)
             end
         else
-            broken[pkg] = true
-            log('Package %s depends on broken %s',
-                pkg, brokenDep(pkg))
+            broken[item] = true
+            log('Item %s depends on broken item %s',
+                item, brokenDep(item))
         end
     end
     return unbroken
 end
 
-local function filterFiles(pkg, filter_common)
-    local list = readFileList(listFile(pkg))
-    local list2 = {}
-    local common_list = COMMON_FILES[pkg]
-    for _, installed_file in ipairs(list) do
-        local listed = isListed(installed_file, common_list)
-        if listed == filter_common then
-            table.insert(list2, installed_file)
-        end
+local function makeDebs(items, item2deps, item2ver)
+    for _, item in ipairs(items) do
+        local deps = assert(item2deps[item], item)
+        local ver = assert(item2ver[item], item)
+        local list_path = listFile(item)
+        makeDeb(item, list_path, deps, ver)
     end
-    return list2
-end
-
-local function excludeCommon(pkg)
-    local noncommon_files = filterFiles(pkg, false)
-    saveFileList(listFile(pkg), noncommon_files)
-end
-
-local function makeCommonDeb(pkg, ver)
-    local common_files = filterFiles(pkg, true)
-    local list_path = ('common-%s.list'):format(pkg)
-    saveFileList(list_path, common_files)
-    local orig_target = target
-    target = 'common'
-    makeDeb(pkg, list_path, {}, ver)
-    target = orig_target
-end
-
-local function makeDebs(pkgs, pkg2deps, pkg2ver)
-    for _, pkg in ipairs(pkgs) do
-        local deps = assert(pkg2deps[pkg], pkg)
-        local ver = assert(pkg2ver[pkg], pkg)
-        local list_path = listFile(pkg)
-        local add_common = false
-        if COMMON_FILES[pkg] then
-            if target == ARCH_FOR_COMMON then
-                makeCommonDeb(pkg, ver)
-            end
-            add_common = true
-            excludeCommon(pkg)
-        end
-        makeDeb(pkg, list_path, deps, ver, add_common)
-    end
-end
-
-local function buildForTarget(mxe_target, file2pkg)
-    target = mxe_target
-    local pkgs, pkg2deps, pkg2ver = getPkgs()
-    local build_list = sortForBuild(pkgs, pkg2deps)
-    if max_packages then
-        while #build_list > max_packages do
-            table.remove(build_list)
-        end
-    end
-    local unbroken = buildPackages(build_list, pkg2deps, file2pkg)
-    makeDebs(unbroken, pkg2deps, pkg2ver)
 end
 
 local function getMxeVersion()
@@ -626,10 +550,11 @@ assert(execute(("%s check-requirements"):format(tool 'make')))
 while not execute(('%s download -j 6 -k'):format(tool 'make')) do
 end
 gitInit()
-local file2pkg = {}
-for _, t in ipairs(TARGETS) do
-    buildForTarget(t, file2pkg)
-end
+local items, item2deps, item2ver = getItems()
+local build_list = sortForBuild(items, item2deps)
+build_list = sliceArray(build_list, max_items)
+local unbroken = buildPackages(build_list, item2deps)
+makeDebs(unbroken, item2deps, item2ver)
 if not no_debs then
     makeMxeRequirementsDeb('wheezy')
     makeMxeRequirementsDeb('jessie')