diff options
Diffstat (limited to 'SCons')
| -rw-r--r-- | SCons/Action.py | 2 | ||||
| -rw-r--r-- | SCons/CacheDirTests.py | 8 | ||||
| -rw-r--r-- | SCons/Defaults.py | 5 | ||||
| -rw-r--r-- | SCons/Environment.py | 4 | ||||
| -rw-r--r-- | SCons/Environment.xml | 5 | ||||
| -rw-r--r-- | SCons/Node/Alias.py | 4 | ||||
| -rw-r--r-- | SCons/Node/FS.py | 22 | ||||
| -rw-r--r-- | SCons/Node/FSTests.py | 10 | ||||
| -rw-r--r-- | SCons/Node/__init__.py | 8 | ||||
| -rw-r--r-- | SCons/SConf.py | 4 | ||||
| -rw-r--r-- | SCons/Script/Main.py | 5 | ||||
| -rw-r--r-- | SCons/Script/SConsOptions.py | 10 | ||||
| -rw-r--r-- | SCons/Util.py | 163 | ||||
| -rw-r--r-- | SCons/UtilTests.py | 43 |
14 files changed, 199 insertions, 94 deletions
diff --git a/SCons/Action.py b/SCons/Action.py index d7f10df..fdb0463 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -54,7 +54,7 @@ other modules: get_contents() Fetches the "contents" of an Action for signature calculation - plus the varlist. This is what gets MD5 checksummed to decide + plus the varlist. This is what gets checksummed to decide if a target needs to be rebuilt because its action changed. genstring() diff --git a/SCons/CacheDirTests.py b/SCons/CacheDirTests.py index f0278ac..c4a0ed7 100644 --- a/SCons/CacheDirTests.py +++ b/SCons/CacheDirTests.py @@ -96,10 +96,10 @@ class CacheDirTestCase(BaseTestCase): # Verify how the cachepath() method determines the name # of the file in cache. - def my_collect(list): + def my_collect(list, hash_format=None): return list[0] - save_collect = SCons.Util.MD5collect - SCons.Util.MD5collect = my_collect + save_collect = SCons.Util.hash_collect + SCons.Util.hash_collect = my_collect try: name = 'a_fake_bsig' @@ -110,7 +110,7 @@ class CacheDirTestCase(BaseTestCase): filename = os.path.join(dirname, name) assert result == (dirname, filename), result finally: - SCons.Util.MD5collect = save_collect + SCons.Util.hash_collect = save_collect class ExceptionTestCase(unittest.TestCase): """Test that the correct exceptions are thrown by CacheDir.""" diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 06d938c..6a87787 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -78,10 +78,7 @@ def DefaultEnvironment(*args, **kw): if not _default_env: import SCons.Util _default_env = SCons.Environment.Environment(*args, **kw) - if SCons.Util.md5: - _default_env.Decider('MD5') - else: - _default_env.Decider('timestamp-match') + _default_env.Decider('content') global DefaultEnvironment DefaultEnvironment = _fetch_DefaultEnvironment _default_env._CacheDir_path = None diff --git a/SCons/Environment.py b/SCons/Environment.py index cc62679..0bf5079 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -1483,10 +1483,8 @@ class Base(SubstitutionEnvironment): def Decider(self, function): copy_function = self._copy2_from_cache if function in ('MD5', 'content'): - if not SCons.Util.md5: - raise UserError("MD5 signatures are not available in this version of Python.") function = self._changed_content - elif function == 'MD5-timestamp': + elif function in ('MD5-timestamp', 'content-timestamp'): function = self._changed_timestamp_then_content elif function in ('timestamp-newer', 'make'): function = self._changed_timestamp_newer diff --git a/SCons/Environment.xml b/SCons/Environment.xml index a1fd8ec..7d3927b 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -1103,6 +1103,9 @@ that runs a build, updates a file, and runs the build again, all within a single second. +<literal>content-timestamp</literal> +can be used as a synonym for +<literal>MD5-timestamp</literal>. </para> </listitem> </varlistentry> @@ -1117,7 +1120,7 @@ Examples: # Use exact timestamp matches by default. Decider('timestamp-match') -# Use MD5 content signatures for any targets built +# Use hash content signatures for any targets built # with the attached construction environment. env.Decider('content') </example_commands> diff --git a/SCons/Node/Alias.py b/SCons/Node/Alias.py index 64054d2..b5e4eb4 100644 --- a/SCons/Node/Alias.py +++ b/SCons/Node/Alias.py @@ -31,7 +31,7 @@ import collections import SCons.Errors import SCons.Node import SCons.Util -from SCons.Util import MD5signature +from SCons.Util import hash_signature class AliasNameSpace(collections.UserDict): def Alias(self, name, **kw): @@ -161,7 +161,7 @@ class Alias(SCons.Node.Node): pass contents = self.get_contents() - csig = MD5signature(contents) + csig = hash_signature(contents) self.get_ninfo().csig = csig return csig diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 967f007..1e5d5b8 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -50,7 +50,7 @@ import SCons.Node import SCons.Node.Alias import SCons.Subst import SCons.Util -from SCons.Util import MD5signature, MD5filesignature, MD5collect +from SCons.Util import hash_signature, hash_file_signature, hash_collect import SCons.Warnings print_duplicate = 0 @@ -1865,7 +1865,7 @@ class Dir(Base): node is called which has a child directory, the child directory should return the hash of its contents.""" contents = self.get_contents() - return MD5signature(contents) + return hash_signature(contents) def do_duplicate(self, src): pass @@ -2630,7 +2630,7 @@ class File(Base): BuildInfo = FileBuildInfo # Although the command-line argument is in kilobytes, this is in bytes. - md5_chunksize = 65536 + hash_chunksize = 65536 def diskcheck_match(self): diskcheck_match(self, self.isdir, @@ -2726,13 +2726,13 @@ class File(Base): def get_content_hash(self): """ - Compute and return the MD5 hash for this file. + Compute and return the hash for this file. """ if not self.rexists(): - return MD5signature('') + return hash_signature('') fname = self.rfile().get_abspath() try: - cs = MD5filesignature(fname, chunksize=File.md5_chunksize) + cs = hash_file_signature(fname, chunksize=File.hash_chunksize) except EnvironmentError as e: if not e.filename: e.filename = fname @@ -3218,7 +3218,7 @@ class File(Base): if csig is None: try: - if self.get_size() < File.md5_chunksize: + if self.get_size() < File.hash_chunksize: contents = self.get_contents() else: csig = self.get_content_hash() @@ -3230,7 +3230,7 @@ class File(Base): csig = '' else: if not csig: - csig = SCons.Util.MD5signature(contents) + csig = SCons.Util.hash_signature(contents) ninfo.csig = csig @@ -3619,7 +3619,7 @@ class File(Base): cachedir, cachefile = self.get_build_env().get_CacheDir().cachepath(self) if not self.exists() and cachefile and os.path.exists(cachefile): - self.cachedir_csig = MD5filesignature(cachefile, File.md5_chunksize) + self.cachedir_csig = hash_file_signature(cachefile, File.hash_chunksize) else: self.cachedir_csig = self.get_csig() return self.cachedir_csig @@ -3639,7 +3639,7 @@ class File(Base): executor = self.get_executor() - result = self.contentsig = MD5signature(executor.get_contents()) + result = self.contentsig = hash_signature(executor.get_contents()) return result def get_cachedir_bsig(self): @@ -3670,7 +3670,7 @@ class File(Base): sigs.append(self.get_internal_path()) # Merge this all into a single signature - result = self.cachesig = MD5collect(sigs) + result = self.cachesig = hash_collect(sigs) return result default_fs = None diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index 65cc363..457d199 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -2055,15 +2055,15 @@ class DirTestCase(_tempdirTestCase): assert g.get_csig() + " g" == files[2], files assert s.get_csig() + " sub" == files[3], files - def test_md5_chunksize(self): + def test_hash_chunksize(self): """ - Test verifying that File.get_csig() correctly uses md5_chunksize. This - variable is documented as the md5 chunksize in kilobytes. This test - verifies that if the file size is less than the md5 chunksize, + Test verifying that File.get_csig() correctly uses hash_chunksize. This + variable is documented as the hash chunksize in kilobytes. This test + verifies that if the file size is less than the hash chunksize, get_contents() is called; otherwise, it verifies that get_contents() is not called. """ - chunksize_bytes = SCons.Node.FS.File.md5_chunksize + chunksize_bytes = SCons.Node.FS.File.hash_chunksize test = self.test test.subdir('chunksize_dir') diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index 8a1677e..6a3770e 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -54,7 +54,7 @@ from SCons.Debug import logInstanceCreation import SCons.Executor import SCons.Memoize import SCons.Util -from SCons.Util import MD5signature +from SCons.Util import hash_signature from SCons.Debug import Trace @@ -1173,7 +1173,7 @@ class Node(object, metaclass=NoSlotsPyPy): if self.has_builder(): binfo.bact = str(executor) - binfo.bactsig = MD5signature(executor.get_contents()) + binfo.bactsig = hash_signature(executor.get_contents()) if self._specific_sources: sources = [s for s in self.sources if s not in ignore_set] @@ -1211,7 +1211,7 @@ class Node(object, metaclass=NoSlotsPyPy): return self.ninfo.csig except AttributeError: ninfo = self.get_ninfo() - ninfo.csig = MD5signature(self.get_contents()) + ninfo.csig = hash_signature(self.get_contents()) return self.ninfo.csig def get_cachedir_csig(self): @@ -1502,7 +1502,7 @@ class Node(object, metaclass=NoSlotsPyPy): if self.has_builder(): contents = self.get_executor().get_contents() - newsig = MD5signature(contents) + newsig = hash_signature(contents) if bi.bactsig != newsig: if t: Trace(': bactsig %s != newsig %s' % (bi.bactsig, newsig)) result = True diff --git a/SCons/SConf.py b/SCons/SConf.py index 70a98c8..d024600 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -598,7 +598,7 @@ class SConfBase: f = "conftest" if text is not None: - textSig = SCons.Util.MD5signature(sourcetext) + textSig = SCons.Util.hash_signature(sourcetext) textSigCounter = str(_ac_build_counter[textSig]) _ac_build_counter[textSig] += 1 @@ -617,7 +617,7 @@ class SConfBase: target = None action = builder.builder.action.get_contents(target=target, source=[source], env=self.env) - actionsig = SCons.Util.MD5signature(action) + actionsig = SCons.Util.hash_signature(action) f = "_".join([f, actionsig]) pref = self.env.subst( builder.builder.prefix ) diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 1f2a455..ea9bd95 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -1098,8 +1098,11 @@ def _main(parser): SCons.Job.explicit_stack_size = options.stack_size + # Hash format and chunksize are set late to support SetOption being called + # in a SConscript or SConstruct file. + SCons.Util.set_hash_format(options.hash_format) if options.md5_chunksize: - SCons.Node.FS.File.md5_chunksize = options.md5_chunksize * 1024 + SCons.Node.FS.File.hash_chunksize = options.md5_chunksize * 1024 platform = SCons.Platform.platform_module() diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index d25d657..cf132a2 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -130,6 +130,7 @@ class SConsValues(optparse.Values): 'clean', 'diskcheck', 'duplicate', + 'hash_format', 'help', 'implicit_cache', 'max_drift', @@ -713,6 +714,11 @@ def Parser(version): action="help", help="Print this message and exit.") + op.add_option('--hash-format', + dest='hash_format', + action='store', + help='Hash format (e.g. md5, sha1, or sha256).') + op.add_option('-i', '--ignore-errors', dest='ignore_errors', default=False, action="store_true", @@ -775,9 +781,9 @@ def Parser(version): op.add_option('--md5-chunksize', nargs=1, type="int", - dest='md5_chunksize', default=SCons.Node.FS.File.md5_chunksize, + dest='md5_chunksize', default=SCons.Node.FS.File.hash_chunksize, action="store", - help="Set chunk-size for MD5 signature computation to N kilobytes.", + help="Set chunk-size for hash signature computation to N kilobytes.", metavar="N") op.add_option('-n', '--no-exec', '--just-print', '--dry-run', '--recon', diff --git a/SCons/Util.py b/SCons/Util.py index 88aeaae..053c5de 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1468,66 +1468,143 @@ def AddMethod(obj, function, name=None): setattr(obj, name, method) -if hasattr(hashlib, 'md5'): - md5 = True +# Default hash function. SCons-internal. +_hash_function = None - def MD5signature(s): - """ - Generate md5 signature of a string - :param s: either string or bytes. Normally should be bytes - :return: String of hex digits representing the signature - """ - m = hashlib.md5() +def set_hash_format(hash_format): + """ + Sets the default hash format used by SCons. If hash_format is None or + an empty string, the default is determined by this function. - try: - m.update(to_bytes(s)) - except TypeError as e: - m.update(to_bytes(str(s))) + Currently the default behavior is to use the first available format of + the following options: MD5, SHA1, SHA256. + """ + global _hash_function + + if hash_format: + hash_format_lower = hash_format.lower() + _hash_function = getattr(hashlib, hash_format_lower, None) + if _hash_function is None: + raise Exception( + 'Hash format "%s" is not available in your Python ' + 'interpreter.' % hash_format_lower) + else: + # Set the default hash format based on what is available, defaulting + # to md5 for backwards compatibility. + choices = ['md5', 'sha1', 'sha256'] + for choice in choices: + try: + _hash_function = getattr(hashlib, choice) + _hash_function() + break + except Exception: + pass + else: + # This is not expected to happen in practice. + raise Exception( + 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' + 'SCons requires at least one.') + +# Ensure that this is initialized in case either: +# 1. This code is running in a unit test. +# 2. This code is running in a consumer that does hash operations while +# SConscript files are being loaded. +# TODO: Should this go somewhere else? Is this unnecessary? Case #1 could be +# handled in the TestCmd module, but I was worried about breaking people +# who mischievously calls get_csig() during startup. +set_hash_format('md5') + + +def _get_hash_object(hash_format): + """ + Allocates a hash object using the requested hash format. - return m.hexdigest() + :param hash_format: Hash format to use. + :return: hashlib object. + """ + if hash_format is None: + if _hash_function is None: + raise Exception('There is no default hash function. Did you call ' + 'a hashing function before SCons was initialized?') + return _hash_function() + elif not hasattr(hashlib, hash_format): + raise Exception( + 'Hash format "%s" is not available in your Python interpreter.' % + hash_format) + else: + return getattr(hashlib, hash_format)() - def MD5filesignature(fname, chunksize=65536): - """ - Generate the md5 signature of a file - :param fname: file to hash - :param chunksize: chunk size to read - :return: String of Hex digits representing the signature - """ - m = hashlib.md5() - with open(fname, "rb") as f: - while True: - blck = f.read(chunksize) - if not blck: - break - m.update(to_bytes(blck)) - return m.hexdigest() -else: - # if md5 algorithm not available, just return data unmodified - # could add alternative signature scheme here - md5 = False +def hash_signature(s, hash_format=None): + """ + Generate hash signature of a string - def MD5signature(s): - return str(s) + :param s: either string or bytes. Normally should be bytes + :param hash_format: Specify to override default hash format + :return: String of hex digits representing the signature + """ + m = _get_hash_object(hash_format) + try: + m.update(to_bytes(s)) + except TypeError: + m.update(to_bytes(str(s))) - def MD5filesignature(fname, chunksize=65536): - with open(fname, "rb") as f: - result = f.read() - return result + return m.hexdigest() -def MD5collect(signatures): +def hash_file_signature(fname, chunksize=65536, hash_format=None): + """ + Generate the md5 signature of a file + + :param fname: file to hash + :param chunksize: chunk size to read + :param hash_format: Specify to override default hash format + :return: String of Hex digits representing the signature + """ + m = _get_hash_object(hash_format) + with open(fname, "rb") as f: + while True: + blck = f.read(chunksize) + if not blck: + break + m.update(to_bytes(blck)) + return m.hexdigest() + + +def hash_collect(signatures, hash_format=None): """ Collects a list of signatures into an aggregate signature. - signatures - a list of signatures - returns - the aggregate signature + :param signatures: a list of signatures + :param hash_format: Specify to override default hash format + :return: - the aggregate signature """ if len(signatures) == 1: return signatures[0] else: - return MD5signature(', '.join(signatures)) + return hash_signature(', '.join(signatures), hash_format) + + +def MD5signature(s): + """ + Deprecated. Use hash_signature instead. + """ + return hash_signature(s) + + +def MD5filesignature(fname, chunksize=65536): + """ + Deprecated. Use hash_file_signature instead. + """ + return hash_file_signature(fname, chunksize) + + +def MD5collect(signatures): + """ + Deprecated. Use hash_collect instead. + """ + return hash_collect(signatures) def silent_intern(x): diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 72bbaf3..2be2672 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -23,6 +23,7 @@ import SCons.compat +import functools import io import os import sys @@ -755,24 +756,44 @@ bling assert id(s1) == id(s4) -class MD5TestCase(unittest.TestCase): +class HashTestCase(unittest.TestCase): def test_collect(self): """Test collecting a list of signatures into a new signature value """ - s = list(map(MD5signature, ('111', '222', '333'))) - - assert '698d51a19d8a121ce581499d7b701668' == MD5collect(s[0:1]) - assert '8980c988edc2c78cc43ccb718c06efd5' == MD5collect(s[0:2]) - assert '53fd88c84ff8a285eb6e0a687e55b8c7' == MD5collect(s) + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + '8980c988edc2c78cc43ccb718c06efd5', + '53fd88c84ff8a285eb6e0a687e55b8c7'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '42eda1b5dcb3586bccfb1c69f22f923145271d97', + '2eb2f7be4e883ebe52034281d818c91e1cf16256'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', + 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), + }.items(): + hs = functools.partial(hash_signature, hash_format=algorithm) + s = list(map(hs, ('111', '222', '333'))) + + assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) + assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) + assert expected[2] == hash_collect(s, hash_format=algorithm) def test_MD5signature(self): """Test generating a signature""" - s = MD5signature('111') - assert '698d51a19d8a121ce581499d7b701668' == s, s - - s = MD5signature('222') - assert 'bcbe3365e6ac95ea2c0343a2395834dd' == s, s + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + 'bcbe3365e6ac95ea2c0343a2395834dd'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '1c6637a8f2e1f75e06ff9984894d6bd16a3a36a9'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), + }.items(): + s = hash_signature('111', hash_format=algorithm) + assert expected[0] == s, s + + s = hash_signature('222', hash_format=algorithm) + assert expected[1] == s, s class NodeListTestCase(unittest.TestCase): |