From fda865786b2d82ee906240ec11c30c36624698c9 Mon Sep 17 00:00:00 2001 From: David H <1810493+djh82@users.noreply.github.com> Date: Wed, 13 Jan 2021 09:59:10 +0000 Subject: feat: Adds ZIP_OVERRIDE_TIMESTAMP --- CHANGES.txt | 5 +++ SCons/Tool/zip.py | 48 ++++++++++++++++++++------- SCons/Tool/zip.xml | 16 +++++++++ test/ZIP/ZIP.py | 41 +++++------------------ test/ZIP/ZIPROOT.py | 46 ++++++++------------------ test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py | 68 ++++++++++++++++++++++++++++++++++++++ testing/framework/TestSCons.py | 36 ++++++++++++++++++-- 7 files changed, 179 insertions(+), 81 deletions(-) create mode 100644 test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py diff --git a/CHANGES.txt b/CHANGES.txt index 133fcb9..c1d7fd9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -43,6 +43,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Change Environment.SideEffect() to not add duplicate side effects. NOTE: The list of returned side effect Nodes will not include any duplicate side effect Nodes. + From David H: + - Add ZIP_OVERRIDE_TIMESTAMP env option to Zip builder which allows for overriding of the file + modification times in the archive. + - Fix Zip builder not rebuilding when ZIPROOT env option was changed. + From Jason Kenny - Fix python3 crash when Value node get_text_content when child content does not have decode() NOTE: If you depend on Value node's get_text_content returning concatenated contents of it's diff --git a/SCons/Tool/zip.py b/SCons/Tool/zip.py index cbf2c16..f3d38fe 100644 --- a/SCons/Tool/zip.py +++ b/SCons/Tool/zip.py @@ -8,8 +8,9 @@ selection method. """ +# MIT License # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -29,39 +30,62 @@ selection method. # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -import os.path +import os import SCons.Builder import SCons.Defaults import SCons.Node.FS import SCons.Util +import time import zipfile + zip_compression = zipfile.ZIP_DEFLATED -def zip(target, source, env): - compression = env.get('ZIPCOMPRESSION', 0) - zf = zipfile.ZipFile(str(target[0]), 'w', compression) +def _create_zipinfo_for_file(fname, arcname, date_time, compression): + st = os.stat(fname) + if not date_time: + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + zinfo = zipfile.ZipInfo(filename=arcname, date_time=date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + zinfo.compress_type = compression + zinfo.file_size = st.st_size + return zinfo + + +def zip_builder(target, source, env): + compression = env.get('ZIPCOMPRESSION', zipfile.ZIP_STORED) + zip_root = str(env.get('ZIPROOT', '')) + date_time = env.get('ZIP_OVERRIDE_TIMESTAMP') + + files = [] for s in source: if s.isdir(): for dirpath, dirnames, filenames in os.walk(str(s)): for fname in filenames: path = os.path.join(dirpath, fname) if os.path.isfile(path): - zf.write(path, os.path.relpath(path, str(env.get('ZIPROOT', '')))) + files.append(path) else: - zf.write(str(s), os.path.relpath(str(s), str(env.get('ZIPROOT', '')))) - zf.close() + files.append(str(s)) + + with zipfile.ZipFile(str(target[0]), 'w', compression) as zf: + for fname in files: + arcname = os.path.relpath(fname, zip_root) + # TODO: Switch to ZipInfo.from_file when 3.6 becomes the base python version + zinfo = _create_zipinfo_for_file(fname, arcname, date_time, compression) + with open(fname, "rb") as f: + zf.writestr(zinfo, f.read()) + # Fix PR #3569 - If you don't specify ZIPCOM and ZIPCOMSTR when creating # env, then it will ignore ZIPCOMSTR set afterwards. -zipAction = SCons.Action.Action(zip, "$ZIPCOMSTR", varlist=['ZIPCOMPRESSION']) +zipAction = SCons.Action.Action(zip_builder, "$ZIPCOMSTR", + varlist=['ZIPCOMPRESSION', 'ZIPROOT', 'ZIP_OVERRIDE_TIMESTAMP']) ZipBuilder = SCons.Builder.Builder(action=SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'), source_factory=SCons.Node.FS.Entry, diff --git a/SCons/Tool/zip.xml b/SCons/Tool/zip.xml index 8cf03f4..f9f01d9 100644 --- a/SCons/Tool/zip.xml +++ b/SCons/Tool/zip.xml @@ -164,4 +164,20 @@ containing a file with the name + + + + +An optional timestamp which overrides the last modification time of +the file when stored inside the Zip archive. This is a tuple of six values: + +Year (>= 1980) +Month (one-based) +Day of month (one-based) +Hours (zero-based) +Minutes (zero-based) +Seconds (zero-based) + + + diff --git a/test/ZIP/ZIP.py b/test/ZIP/ZIP.py index f842caf..b79173c 100644 --- a/test/ZIP/ZIP.py +++ b/test/ZIP/ZIP.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import stat @@ -33,30 +32,6 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() -try: - import zipfile -except ImportError: - x = "Python version has no 'ziplib' module; skipping tests.\n" - test.skip_test(x) - -def zipfile_contains(zipfilename, names): - """Returns True if zipfilename contains all the names, False otherwise.""" - zf=zipfile.ZipFile(zipfilename, 'r') - if type(names)==type(''): - names=[names] - for name in names: - try: - info=zf.getinfo(name) - except KeyError as e: # name not found - zf.close() - return False - return True - -def zipfile_files(fname): - """Returns all the filenames in zip file fname.""" - zf = zipfile.ZipFile(fname, 'r') - return [x.filename for x in zf.infolist()] - test.subdir('sub1') test.write('SConstruct', """ @@ -78,12 +53,12 @@ test.write(['sub1', 'file6'], "sub1/file6\n") test.run(arguments = 'aaa.zip', stderr = None) test.must_exist('aaa.zip') -test.fail_test(not zipfile_contains('aaa.zip', ['file1', 'file2', 'file3'])) +test.fail_test(not test.zipfile_contains('aaa.zip', ['file1', 'file2', 'file3'])) test.run(arguments = 'bbb.zip', stderr = None) test.must_exist('bbb.zip') -test.fail_test(not zipfile_contains('bbb.zip', ['sub1/file5', 'sub1/file6', 'file4'])) +test.fail_test(not test.zipfile_contains('bbb.zip', ['sub1/file5', 'sub1/file6', 'file4'])) ###### @@ -136,11 +111,11 @@ test.run(arguments = '.', stderr = None) test.must_not_exist(test.workpath('f3.zip')) test.must_exist(test.workpath('f3.xyzzy')) -test.fail_test(zipfile_files("f1.zip") != ['file10', 'file11', 'file12']) +test.fail_test(test.zipfile_files("f1.zip") != ['file10', 'file11', 'file12']) -test.fail_test(zipfile_files("f2.zip") != ['file13', 'file14', 'file15']) +test.fail_test(test.zipfile_files("f2.zip") != ['file13', 'file14', 'file15']) -test.fail_test(zipfile_files("f3.xyzzy") != ['file16', 'file17', 'file18']) +test.fail_test(test.zipfile_files("f3.xyzzy") != ['file16', 'file17', 'file18']) f4_size = os.stat('f4.zip')[stat.ST_SIZE] f4stored_size = os.stat('f4stored.zip')[stat.ST_SIZE] diff --git a/test/ZIP/ZIPROOT.py b/test/ZIP/ZIPROOT.py index 24e82eb..10e01d9 100644 --- a/test/ZIP/ZIPROOT.py +++ b/test/ZIP/ZIPROOT.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -33,23 +32,6 @@ test = TestSCons.TestSCons() import zipfile -def zipfile_contains(zipfilename, names): - """Returns True if zipfilename contains all the names, False otherwise.""" - zf=zipfile.ZipFile(zipfilename, 'r') - if type(names)==type(''): - names=[names] - for name in names: - try: - info=zf.getinfo(name) - except KeyError as e: # name not found - zf.close() - return False - return True - -def zipfile_files(fname): - """Returns all the filenames in zip file fname.""" - zf = zipfile.ZipFile(fname, 'r') - return [x.filename for x in zf.infolist()] test.subdir('sub1') test.subdir(['sub1', 'sub2']) @@ -69,13 +51,12 @@ test.run(arguments = 'aaa.zip', stderr = None) test.must_exist('aaa.zip') # TEST: Zip file should contain 'file1', not 'sub1/file1', because of ZIPROOT. -zf=zipfile.ZipFile('aaa.zip', 'r') -test.fail_test(zf.testzip() is not None) -zf.close() +with zipfile.ZipFile('aaa.zip', 'r') as zf: + test.fail_test(zf.testzip() is not None) -files=zipfile_files('aaa.zip') -test.fail_test(zipfile_files('aaa.zip') != ['file1'], - message='Zip file aaa.zip has wrong files: %s'%repr(files)) +files = test.zipfile_files('aaa.zip') +test.fail_test(test.zipfile_files('aaa.zip') != ['file1'], + message='Zip file aaa.zip has wrong files: %s' % repr(files)) ### @@ -84,13 +65,12 @@ test.run(arguments = 'bbb.zip', stderr = None) test.must_exist('bbb.zip') # TEST: Zip file should contain 'sub2/file2', not 'sub1/sub2/file2', because of ZIPROOT. -zf=zipfile.ZipFile('bbb.zip', 'r') -test.fail_test(zf.testzip() is not None) -zf.close() +with zipfile.ZipFile('bbb.zip', 'r') as zf: + test.fail_test(zf.testzip() is not None) -files=zipfile_files('bbb.zip') -test.fail_test(zipfile_files('bbb.zip') != ['file2', 'sub2/file2'], - message='Zip file bbb.zip has wrong files: %s'%repr(files)) +files = test.zipfile_files('bbb.zip') +test.fail_test(test.zipfile_files('bbb.zip') != ['file2', 'sub2/file2'], + message='Zip file bbb.zip has wrong files: %s' % repr(files)) test.pass_test() diff --git a/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py b/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py new file mode 100644 index 0000000..8904944 --- /dev/null +++ b/test/ZIP/ZIP_OVERRIDE_TIMESTAMP.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +import TestSCons + +_python_ = TestSCons._python_ + +test = TestSCons.TestSCons() + +import zipfile + + +def zipfile_get_file_datetime(zipfilename, fname): + """Returns the date_time of the given file with the archive.""" + with zipfile.ZipFile(zipfilename, 'r') as zf: + for info in zf.infolist(): + if info.filename == fname: + return info.date_time + + raise Exception("Unable to find %s" % fname) + + +test.write('SConstruct', """ +env = Environment(tools = ['zip']) +env.Zip(target = 'aaa.zip', source = ['file1'], ZIP_OVERRIDE_TIMESTAMP=(1983,3,11,1,2,2)) +""" % locals()) + +test.write(['file1'], "file1\n") + +test.run(arguments = 'aaa.zip', stderr = None) + +test.must_exist('aaa.zip') + +with zipfile.ZipFile('aaa.zip', 'r') as zf: + test.fail_test(zf.testzip() is not None) + +files = test.zipfile_files('aaa.zip') +test.fail_test(files != ['file1'], + message='Zip file aaa.zip has wrong files: %s' % repr(files)) + +date_time = zipfile_get_file_datetime('aaa.zip', 'file1') +test.fail_test(date_time != (1983, 3, 11, 1, 2, 2), + message="Zip file aaa.zip#file1 has wrong date_time: %s" % repr(date_time)) + +test.pass_test() diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 36b9df4..d1143e0 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -12,9 +12,28 @@ from those classes, as well as any overridden or additional methods or attributes defined in this subclass. """ -# __COPYRIGHT__ - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os import re @@ -22,6 +41,7 @@ import shutil import sys import time import subprocess +import zipfile from collections import namedtuple from TestCommon import * @@ -1604,6 +1624,16 @@ else: else: return True + def zipfile_contains(self, zipfilename, names): + """Returns True if zipfilename contains all the names, False otherwise.""" + with zipfile.ZipFile(zipfilename, 'r') as zf: + return all(elem in zf.namelist() for elem in names) + + def zipfile_files(self, fname): + """Returns all the filenames in zip file fname.""" + with zipfile.ZipFile(fname, 'r') as zf: + return zf.namelist() + class Stat: def __init__(self, name, units, expression, convert=None): -- cgit v0.12