diff options
author | Tony Theodore <tonyt@logyst.com> | 2016-04-30 11:06:09 (GMT) |
---|---|---|
committer | Tony Theodore <tonyt@logyst.com> | 2016-04-30 11:06:09 (GMT) |
commit | ab6b7d9f05de8b5c5edf7323892b0a03df7e41d2 (patch) | |
tree | 224cf6bace91c70a5ee41b2fd07dc21a0b37dd70 | |
parent | 7fb90eda0fb051e9753d2c45f2a5e1c2de3733f4 (diff) | |
parent | c5c610b0853821a8ba824ff7a96d3bfc5d8562e7 (diff) | |
download | mxe-ab6b7d9f05de8b5c5edf7323892b0a03df7e41d2.zip mxe-ab6b7d9f05de8b5c5edf7323892b0a03df7e41d2.tar.gz mxe-ab6b7d9f05de8b5c5edf7323892b0a03df7e41d2.tar.bz2 |
Merge pull request #1243 from LuaAndC/build-pkg-second-pass
build-pkg: build a package second time in tree with all packages
-rwxr-xr-x | tools/build-pkg.lua | 267 |
1 files changed, 200 insertions, 67 deletions
diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index d087031..0af1d33 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -19,6 +19,9 @@ To prevent build-pkg from creating deb packages, set environment variable MXE_NO_DEBS to 1 In this case fakeroot and dpkg-deb are not needed. +To switch off the second pass, set MXE_NO_SECOND_PASS to 1. +See https://github.com/mxe/mxe/issues/1111 + To limit number of packages being built to x, set environment variable MXE_MAX_ITEMS to x, @@ -32,6 +35,7 @@ How to remove them: http://stackoverflow.com/a/4262545 local max_items = tonumber(os.getenv('MXE_MAX_ITEMS')) local no_debs = os.getenv('MXE_NO_DEBS') +local no_second_pass = os.getenv('MXE_NO_SECOND_PASS') local TODAY = os.date("%Y%m%d") @@ -174,6 +178,10 @@ local function fileExists(name) end end +local function isSymlink(name) + return shell(("ls -l %q"):format(name)):sub(1, 1) == "l" +end + local function writeFile(filename, data) local file = io.open(filename, 'w') file:write(data) @@ -309,6 +317,18 @@ local function sortForBuild(items, item2deps) return build_list end +local function isDependency(item, dependency, item2deps) + for _, dep in ipairs(item2deps[item]) do + if dep == dependency then + return true + end + if isDependency(dep, dependency, item2deps) then + return true + end + end + return false +end + local function makeItem2Index(build_list) local item2index = {} for index, item in ipairs(build_list) do @@ -350,10 +370,10 @@ local function isBlacklisted(file) end local GIT_INITIAL = 'initial' -local GIT_ALL = 'first-all' +local GIT_ALL_PSEUDOITEM = 'all' -local function itemToBranch(item) - return 'first-' .. item:gsub('~', '_') +local function itemToBranch(item, pass) + return pass .. '-' .. item:gsub('~', '_') end -- creates git repo in ./usr @@ -383,10 +403,10 @@ local function gitCommit(message) assert(execute(cmd:format(message))) end -local function gitCheckout(new_branch, deps, item2index) +local function gitCheckout(new_branch, deps, item2index, pass_of_deps) local main_dep = deps[1] if main_dep then - main_dep = itemToBranch(main_dep) + main_dep = itemToBranch(main_dep, pass_of_deps) else main_dep = GIT_INITIAL end @@ -398,7 +418,7 @@ local function gitCheckout(new_branch, deps, item2index) local cmd2 = '%s %s merge -q %s -m %q' if not execute(cmd2:format(GIT, GIT_USER, - itemToBranch(deps[i]), + itemToBranch(deps[i], pass_of_deps), message)) then -- probably merge conflict @@ -426,18 +446,19 @@ local function gitCheckout(new_branch, deps, item2index) end local function gitAdd() - os.execute(GIT .. 'add .') + os.execute(GIT .. 'add --all .') end -- return two lists of filepaths under ./usr/ -- 1. new files -- 2. changed files -local function gitStatus() +local function gitStatus(item, item2deps, file2item) local new_files = {} local changed_files = {} local git_st = io.popen(GIT .. 'status --porcelain', 'r') for line in git_st:lines() do local status, file = line:match('(..) (.*)') + assert(status:sub(2, 2) == ' ') status = trim(status) if file:sub(1, 1) == '"' then -- filename with a space is quoted by git @@ -445,7 +466,23 @@ local function gitStatus() end file = 'usr/' .. file if not fileExists(file) then - log('Missing file: %q', file) + if status == 'D' then + local prev_owner = assert(file2item[file]) + if prev_owner == item then + log('Item %s removed %q installed by itself', + item, file) + elseif isDependency(prev_owner, item, item2deps) then + log('Item %s removed %q installed by its follower %s', + item, file, prev_owner) + else + log('Item %s removed %q installed by %s', + item, file, prev_owner) + end + elseif isSymlink(file) then + log('Broken symlink: %q', file) + else + log('Missing file: %q', file) + end elseif not isBlacklisted(file) then if status == 'A' then table.insert(new_files, file) @@ -531,13 +568,16 @@ end local function removeEmptyDirs(item) -- removing an empty dir can reveal another one (parent) + -- don't pass item to mute the log message local go_on = true while go_on do go_on = false local f = io.popen('find usr/* -empty -type d', 'r') for dir in f:lines() do - log("Remove empty directory %s created by %s", - dir, item) + if item then + log("Remove empty directory %s created by %s", + dir, item) + end os.remove(dir) go_on = true end @@ -545,20 +585,90 @@ local function removeEmptyDirs(item) end end +local function prepareTree(pass, item, item2deps, prev_files, item2index) + if pass == 'first' then + gitCheckout( + itemToBranch(item, pass), + item2deps[item], + item2index, + pass + ) + elseif pass == 'second' then + -- Build item second time to check if it builds correctly if + -- its followers and unrelated packages have been built. + gitCheckout( + itemToBranch(item, 'second'), + {GIT_ALL_PSEUDOITEM}, + item2index, + 'first' + ) + removeEmptyDirs() + if prev_files then + -- Remove files of item from previous build. + for _, file in ipairs(prev_files) do + os.remove(file) + end + gitAdd() + gitCommit(("Remove %s to rebuild it"):format(item, pass)) + end + else + error("Unknown pass: " .. pass) + end +end + +local function comparePasses(item, new_files, prev_file2item, prev_files) + local files_set = {} + for _, file in ipairs(new_files) do + if not prev_file2item[file] then + log('Item %s installs a file on second pass only: %s', + item, file) + elseif prev_file2item[file] ~= item then + log('File %s was installed by %s on first pass ' .. + 'and by %s - on the second pass', + file, prev_file2item[file], item) + end + files_set[file] = true + end + for _, file in ipairs(prev_files) do + if not files_set[file] then + log('Item %s installs a file on first pass only: %s', + item, file) + end + end + -- TODO compare contents of files (nm for binaries) +end + +local function isBuilt(item, files) + local target, pkg = parseItem(item) + local INSTALLED = 'usr/%s/installed/%s' + local installed = INSTALLED:format(target, pkg) + for _, file in ipairs(files) do + if file == installed then + return true + end + end + return false +end + -- builds package, returns list of new files -local function buildItem(item, item2deps, file2item, item2index) - gitCheckout(itemToBranch(item), item2deps[item], item2index) +-- prev_files is passed only to second pass. +local function buildItem(item, item2deps, file2item, item2index, pass, prev_files) + prepareTree(pass, item, item2deps, prev_files, item2index) local target, pkg = parseItem(item) local cmd = '%s %s MXE_TARGETS=%s --jobs=1' os.execute(cmd:format(tool 'make', pkg, target)) gitAdd() - local new_files, changed_files = gitStatus() + local new_files, changed_files = gitStatus(item, item2deps, file2item) if #new_files + #changed_files > 0 then - gitCommit(("Build %s"):format(item)) + gitCommit(("Build %s, pass %s"):format(item, pass)) end - for _, file in ipairs(new_files) do - checkFile(file, item) - file2item[file] = item + if pass == 'first' then + for _, file in ipairs(new_files) do + checkFile(file, item) + file2item[file] = item + end + elseif isBuilt(item, new_files) then + comparePasses(item, new_files, file2item, prev_files) end for _, file in ipairs(changed_files) do checkFile(file, item) @@ -567,8 +677,8 @@ local function buildItem(item, item2deps, file2item, item2index) if not isInArray(creator_item, item2deps[item]) then table.insert(item2deps[item], creator_item) end - log('Item %s changes %s, created by %s', - item, file, creator_item) + log('Item %s (pass %s) changes %s, created by %s', + item, pass, file, creator_item) end checkFileList(concatArrays(new_files, changed_files), item) removeEmptyDirs(item) @@ -687,18 +797,6 @@ local function makeDeb(item, files, deps, ver) makePackage(deb_pkg, files, deb_deps, ver, d1, d2) end -local function isBuilt(item, files) - local target, pkg = parseItem(item) - local INSTALLED = 'usr/%s/installed/%s' - local installed = INSTALLED:format(target, pkg) - for _, file in ipairs(files) do - if file == installed then - return true - end - end - return false -end - local function findForeignInstalls(item, files) for _, file in ipairs(files) do local pattern = 'usr/([^/]+)/installed/([^/]+)' @@ -771,7 +869,8 @@ local function isEmpty(files) end -- build all packages, save filelist to list file -local function buildPackages(items, item2deps) +-- prev_files is passed only to second pass. +local function buildPackages(items, item2deps, pass, prev_item2files) local broken = {} local unbroken = {} local file2item = {} @@ -784,12 +883,23 @@ local function buildPackages(items, item2deps) end return false end + if pass == 'second' then + assert(prev_item2files) + -- fill file2item with data from prev_item2files + for item, files in pairs(prev_item2files) do + for _, file in ipairs(files) do + file2item[file] = item + end + end + end local item2index = makeItem2Index(items) local progress_printer = progressPrinter(items) for i, item in ipairs(items) do if not brokenDep(item) then - local files = buildItem(item, item2deps, - file2item, item2index) + local prev_files = prev_item2files and prev_item2files[item] + local files = buildItem( + item, item2deps, file2item, item2index, pass, prev_files + ) findForeignInstalls(item, files) if isBuilt(item, files) then item2files[item] = files @@ -917,36 +1027,59 @@ local function makeMxeSourcePackage() makePackage(name, files, deps, ver, d1, d2) end -assert(not io.open('usr/.git'), 'Remove usr/') -local MXE_DIR_EXPECTED = '/usr/lib/mxe' -if MXE_DIR ~= MXE_DIR_EXPECTED then - log("Warning! Building in dir %s, not in %s", - MXE_DIR, MXE_DIR_EXPECTED) -end -gitInit() -assert(execute(("%s check-requirements MXE_TARGETS=%q"):format( - tool 'make', table.concat(TARGETS, ' ')))) -if not max_items then - local cmd = ('%s download -j 6 -k'):format(tool 'make') - while not execute(cmd) do end -end -gitAdd() -gitCommit('Initial commit') -gitTag(GIT_INITIAL) -local items, item2deps, item2ver = getItems() -local build_list = sortForBuild(items, item2deps) -assert(isTopoOrdered(build_list, items, item2deps)) -build_list = sliceArray(build_list, max_items) -local unbroken, item2files = buildPackages(build_list, item2deps) -gitCheckout(GIT_ALL, unbroken, makeItem2Index(build_list)) -makeDebs(unbroken, item2deps, item2ver, item2files) -if not no_debs then - makeMxeRequirementsPackage('wheezy') - makeMxeRequirementsPackage('jessie') -end -makeMxeSourcePackage() -if #unbroken < #build_list then - local code = 1 - local close = true - os.exit(code, close) +local function main() + assert(not io.open('usr/.git'), 'Remove usr/') + local MXE_DIR_EXPECTED = '/usr/lib/mxe' + if MXE_DIR ~= MXE_DIR_EXPECTED then + log("Warning! Building in dir %s, not in %s", + MXE_DIR, MXE_DIR_EXPECTED) + end + gitInit() + assert(execute(("%s check-requirements MXE_TARGETS=%q"):format( + tool 'make', table.concat(TARGETS, ' ')))) + if not max_items then + local cmd = ('%s download -j 6 -k'):format(tool 'make') + while not execute(cmd) do end + end + gitAdd() + gitCommit('Initial commit') + gitTag(GIT_INITIAL) + local items, item2deps, item2ver = getItems() + local build_list = sortForBuild(items, item2deps) + assert(isTopoOrdered(build_list, items, item2deps)) + build_list = sliceArray(build_list, max_items) + local first_pass_failed, second_pass_failed + local unbroken, item2files = buildPackages( + build_list, item2deps, 'first' + ) + if #unbroken < #build_list then + first_pass_failed = true + end + gitCheckout( + itemToBranch(GIT_ALL_PSEUDOITEM, 'first'), + unbroken, + makeItem2Index(build_list), + 'first' + ) + makeDebs(unbroken, item2deps, item2ver, item2files) + if not no_debs then + makeMxeRequirementsPackage('wheezy') + makeMxeRequirementsPackage('jessie') + end + makeMxeSourcePackage() + if not no_second_pass then + local unbroken_second = buildPackages( + build_list, item2deps, 'second', item2files + ) + if #unbroken_second < #build_list then + second_pass_failed = true + end + end + if first_pass_failed or second_pass_failed then + local code = 1 + local close = true + os.exit(code, close) + end end + +main() |