diff options
| author | William Deegan <bill@baddogconsulting.com> | 2021-11-16 18:46:46 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-11-16 18:46:46 (GMT) |
| commit | 02f4075bf491267bbc872587d9845086560fc6b1 (patch) | |
| tree | a08b4a7f3e6397daf20615f13e5c84d2ebd0726e | |
| parent | cd98aff49191140710cec03696efbb62281bf416 (diff) | |
| parent | 3b9bcab73ec204e5228b7ad8990e5db06ffad155 (diff) | |
| download | SCons-02f4075bf491267bbc872587d9845086560fc6b1.zip SCons-02f4075bf491267bbc872587d9845086560fc6b1.tar.gz SCons-02f4075bf491267bbc872587d9845086560fc6b1.tar.bz2 | |
Merge pull request #4047 from jcassagnol-public/compile-with-fips-enabled
Fix tests to work with FIPS enabled
38 files changed, 713 insertions, 214 deletions
@@ -34,7 +34,7 @@ venv.bak/ # SCons files -.sconsign.* +.sconsign* # Tool output .coverage diff --git a/CHANGES.txt b/CHANGES.txt index 0dd0cf3..2f5c5c2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,14 @@ NOTE: The 4.2.0 Release of SCons will deprecate Python 3.5 Support. Python 3.5 s RELEASE VERSION/DATE TO BE FILLED IN LATER + From Jacob Cassagnol: + - Default hash algorithm check updated for SCons FIPS compliance. Now checks for hash viability + first and then walks the tree to use the first viable hash as the default one. This typically + selects SHA1 on FIPS-enabled systems less than Python 3.9 as the new default instead of MD5, + unless SHA1 has also been disabled by security policy, at which point SCons selects SHA256 + as the default. For systems running Python 3.9 and later, the hashlib bug has been fixed, + and SCons will once again default to MD5 as the preferred algorithm. + From Joseph Brill: - Fix MSVS tests (vs-N.N-exec.py) for MSVS 6.0, 7.0, and 7.1 (import missing module). - Add support for Visual Studio 2022. diff --git a/SCons/Environment.py b/SCons/Environment.py index 957a6e4..c8cddf7 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -2252,7 +2252,7 @@ class Base(SubstitutionEnvironment): nkw = self.subst_kw(kw) return SCons.Scanner.ScannerBase(*nargs, **nkw) - def SConsignFile(self, name=".sconsign", dbm_module=None): + def SConsignFile(self, name=SCons.SConsign.current_sconsign_filename(), dbm_module=None): if name is not None: name = self.subst(name) if not os.path.isabs(name): diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index 2ed897a..cd36fd2 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -41,6 +41,7 @@ from SCons.Environment import ( is_valid_construction_var, ) from SCons.Util import CLVar +from SCons.SConsign import current_sconsign_filename def diff_env(env1, env2): @@ -3321,7 +3322,7 @@ def generate(env): assert dbms[-1] == 7, dbms env.SConsignFile() - assert fnames[-1] == os.path.join(os.sep, 'dir', '.sconsign'), fnames + assert fnames[-1] == os.path.join(os.sep, 'dir', current_sconsign_filename()), fnames assert dbms[-1] is None, dbms env.SConsignFile(None) diff --git a/SCons/Node/NodeTests.py b/SCons/Node/NodeTests.py index 29a3887..b36c2e9 100644 --- a/SCons/Node/NodeTests.py +++ b/SCons/Node/NodeTests.py @@ -592,7 +592,7 @@ class NodeTestCase(unittest.TestCase): node = SCons.Node.Node() node._func_get_contents = 4 result = node.get_csig() - assert result == '550a141f12de6341fba65b0ad0433500', result + assert result in ('550a141f12de6341fba65b0ad0433500', '9a3e61b6bcc8abec08f195526c3132d5a4a98cc0', '3538a1ef2e113da64249eea7bd068b585ec7ce5df73b2d1e319d8c9bf47eb314'), result finally: SCons.Node.Node.NodeInfo = SCons.Node.NodeInfoBase @@ -609,7 +609,7 @@ class NodeTestCase(unittest.TestCase): node = SCons.Node.Node() node._func_get_contents = 4 result = node.get_cachedir_csig() - assert result == '15de21c670ae7c3f6f3f1f37029303c9', result + assert result in ('15de21c670ae7c3f6f3f1f37029303c9', 'cfa1150f1787186742a9a884b73a43d8cf219f9b', '91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8'), result finally: SCons.Node.Node.NodeInfo = SCons.Node.NodeInfoBase diff --git a/SCons/SConsign.py b/SCons/SConsign.py index ade8a10..5b78855 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -58,16 +58,25 @@ DB_Module = SCons.dblite DB_Name = None DB_sync_list = [] +def current_sconsign_filename(): + hash_format = SCons.Util.get_hash_format() + current_hash_algorithm = SCons.Util.get_current_hash_algorithm_used() + # if the user left the options defaulted AND the default algorithm set by + # SCons is md5, then set the database name to be the special default name + # + # otherwise, if it defaults to something like 'sha1' or the user explicitly + # set 'md5' as the hash format, set the database name to .sconsign_<algorithm> + # eg .sconsign_sha1, etc. + if hash_format is None and current_hash_algorithm == 'md5': + return ".sconsign" + else: + return ".sconsign_" + current_hash_algorithm def Get_DataBase(dir): global DataBase, DB_Module, DB_Name if DB_Name is None: - hash_format = SCons.Util.get_hash_format() - if hash_format is None: - DB_Name = ".sconsign" - else: - DB_Name = ".sconsign_%s" % hash_format + DB_Name = current_sconsign_filename() top = dir.fs.Top if not os.path.isabs(DB_Name) and top.repositories: @@ -335,7 +344,7 @@ class DirFile(Dir): """ self.dir = dir - self.sconsign = os.path.join(dir.get_internal_path(), '.sconsign') + self.sconsign = os.path.join(dir.get_internal_path(), current_sconsign_filename()) try: fp = open(self.sconsign, 'rb') diff --git a/SCons/SConsignTests.py b/SCons/SConsignTests.py index d4f4418..e9f2071 100644 --- a/SCons/SConsignTests.py +++ b/SCons/SConsignTests.py @@ -28,6 +28,7 @@ import TestCmd import SCons.dblite import SCons.SConsign +from SCons.Util import get_hash_format, get_current_hash_algorithm_used class BuildInfo: def merge(self, object): @@ -295,7 +296,10 @@ class SConsignFileTestCase(SConsignTestCase): file = test.workpath('sconsign_file') assert SCons.SConsign.DataBase == {}, SCons.SConsign.DataBase - assert SCons.SConsign.DB_Name == ".sconsign", SCons.SConsign.DB_Name + if get_hash_format() is None and get_current_hash_algorithm_used() == 'md5': + assert SCons.SConsign.DB_Name == ".sconsign", SCons.SConsign.DB_Name + else: + assert SCons.SConsign.DB_Name == ".sconsign_{}".format(get_current_hash_algorithm_used()), SCons.SConsign.DB_Name assert SCons.SConsign.DB_Module is SCons.dblite, SCons.SConsign.DB_Module SCons.SConsign.File(file) diff --git a/SCons/Scanner/Dir.py b/SCons/Scanner/Dir.py index 162e9ef..239b1ec 100644 --- a/SCons/Scanner/Dir.py +++ b/SCons/Scanner/Dir.py @@ -63,6 +63,25 @@ skip_entry_list = [ '.sconsign_md5.dblite', '.sconsign_sha1.dblite', '.sconsign_sha256.dblite', + # and all the duplicate files for each sub-sconsfile type + '.sconsign_md5', + '.sconsign_md5.dir', + '.sconsign_md5.pag', + '.sconsign_md5.dat', + '.sconsign_md5.bak', + '.sconsign_md5.db', + '.sconsign_sha1', + '.sconsign_sha1.dir', + '.sconsign_sha1.pag', + '.sconsign_sha1.dat', + '.sconsign_sha1.bak', + '.sconsign_sha1.db', + '.sconsign_sha256', + '.sconsign_sha256.dir', + '.sconsign_sha256.pag', + '.sconsign_sha256.dat', + '.sconsign_sha256.bak', + '.sconsign_sha256.db', ] for skip in skip_entry_list: diff --git a/SCons/Scanner/DirTests.py b/SCons/Scanner/DirTests.py index 2e73fe2..1c46c6c 100644 --- a/SCons/Scanner/DirTests.py +++ b/SCons/Scanner/DirTests.py @@ -28,6 +28,7 @@ import TestCmd import SCons.Node.FS import SCons.Scanner.Dir +from SCons.SConsign import current_sconsign_filename #class DummyNode: # def __init__(self, name, fs): @@ -57,23 +58,25 @@ class DirScannerTestBase(unittest.TestCase): self.test.subdir('dir', ['dir', 'sub']) + sconsign = current_sconsign_filename() + self.test.write(['dir', 'f1'], "dir/f1\n") self.test.write(['dir', 'f2'], "dir/f2\n") - self.test.write(['dir', '.sconsign'], "dir/.sconsign\n") - self.test.write(['dir', '.sconsign.bak'], "dir/.sconsign.bak\n") - self.test.write(['dir', '.sconsign.dat'], "dir/.sconsign.dat\n") - self.test.write(['dir', '.sconsign.db'], "dir/.sconsign.db\n") - self.test.write(['dir', '.sconsign.dblite'], "dir/.sconsign.dblite\n") - self.test.write(['dir', '.sconsign.dir'], "dir/.sconsign.dir\n") - self.test.write(['dir', '.sconsign.pag'], "dir/.sconsign.pag\n") + self.test.write(['dir', '{}'.format(sconsign)], "dir/{}\n".format(sconsign)) + self.test.write(['dir', '{}.bak'.format(sconsign)], "dir/{}.bak\n".format(sconsign)) + self.test.write(['dir', '{}.dat'.format(sconsign)], "dir/{}.dat\n".format(sconsign)) + self.test.write(['dir', '{}.db'.format(sconsign)], "dir/{}.db\n".format(sconsign)) + self.test.write(['dir', '{}.dblite'.format(sconsign)], "dir/{}.dblite\n".format(sconsign)) + self.test.write(['dir', '{}.dir'.format(sconsign)], "dir/{}.dir\n".format(sconsign)) + self.test.write(['dir', '{}.pag'.format(sconsign)], "dir/{}.pag\n".format(sconsign)) self.test.write(['dir', 'sub', 'f3'], "dir/sub/f3\n") self.test.write(['dir', 'sub', 'f4'], "dir/sub/f4\n") - self.test.write(['dir', 'sub', '.sconsign'], "dir/.sconsign\n") - self.test.write(['dir', 'sub', '.sconsign.bak'], "dir/.sconsign.bak\n") - self.test.write(['dir', 'sub', '.sconsign.dat'], "dir/.sconsign.dat\n") - self.test.write(['dir', 'sub', '.sconsign.dblite'], "dir/.sconsign.dblite\n") - self.test.write(['dir', 'sub', '.sconsign.dir'], "dir/.sconsign.dir\n") - self.test.write(['dir', 'sub', '.sconsign.pag'], "dir/.sconsign.pag\n") + self.test.write(['dir', 'sub', '{}'.format(sconsign)], "dir/{}\n".format(sconsign)) + self.test.write(['dir', 'sub', '{}.bak'.format(sconsign)], "dir/{}.bak\n".format(sconsign)) + self.test.write(['dir', 'sub', '{}.dat'.format(sconsign)], "dir/{}.dat\n".format(sconsign)) + self.test.write(['dir', 'sub', '{}.dblite'.format(sconsign)], "dir/{}.dblite\n".format(sconsign)) + self.test.write(['dir', 'sub', '{}.dir'.format(sconsign)], "dir/{}.dir\n".format(sconsign)) + self.test.write(['dir', 'sub', '{}.pag'.format(sconsign)], "dir/{}.pag\n".format(sconsign)) class DirScannerTestCase(DirScannerTestBase): def runTest(self): @@ -88,7 +91,7 @@ class DirScannerTestCase(DirScannerTestBase): ] deps = s(env.Dir('dir'), env, ()) sss = list(map(str, deps)) - assert sss == expect, sss + assert sss == expect, "Found {}, expected {}".format(sss, expect) expect = [ os.path.join('dir', 'sub', 'f3'), @@ -96,7 +99,7 @@ class DirScannerTestCase(DirScannerTestBase): ] deps = s(env.Dir('dir/sub'), env, ()) sss = list(map(str, deps)) - assert sss == expect, sss + assert sss == expect, "Found {}, expected {}".format(sss, expect) class DirEntryScannerTestCase(DirScannerTestBase): def runTest(self): @@ -106,16 +109,16 @@ class DirEntryScannerTestCase(DirScannerTestBase): deps = s(env.Dir('dir'), env, ()) sss = list(map(str, deps)) - assert sss == [], sss + assert sss == [], "Found {}, expected {}".format(sss, []) deps = s(env.Dir('dir/sub'), env, ()) sss = list(map(str, deps)) - assert sss == [], sss + assert sss == [], "Found {}, expected {}".format(sss, []) # Make sure we don't blow up if handed a non-Dir node. deps = s(env.File('dir/f1'), env, ()) sss = list(map(str, deps)) - assert sss == [], sss + assert sss == [], "Found {}, expected {}".format(sss, []) if __name__ == "__main__": unittest.main() diff --git a/SCons/Util.py b/SCons/Util.py index 0dd6ff1..cbd99b0 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1669,10 +1669,86 @@ def AddMethod(obj, function, name=None): # Default hash function and format. SCons-internal. -ALLOWED_HASH_FORMATS = ['md5', 'sha1', 'sha256'] +DEFAULT_HASH_FORMATS = ['md5', 'sha1', 'sha256'] +ALLOWED_HASH_FORMATS = [] _HASH_FUNCTION = None _HASH_FORMAT = None +def _attempt_init_of_python_3_9_hash_object(hash_function_object, sys_used=sys): + """Python 3.9 and onwards lets us initialize the hash function object with the + key "usedforsecurity"=false. This lets us continue to use algorithms that have + been deprecated either by FIPS or by Python itself, as the MD5 algorithm SCons + prefers is not being used for security purposes as much as a short, 32 char + hash that is resistant to accidental collisions. + + In prior versions of python, hashlib returns a native function wrapper, which + errors out when it's queried for the optional parameter, so this function + wraps that call. + + It can still throw a ValueError if the initialization fails due to FIPS + compliance issues, but that is assumed to be the responsibility of the caller. + """ + if hash_function_object is None: + return None + + # https://stackoverflow.com/a/11887885 details how to check versions with the "packaging" library. + # however, for our purposes checking the version is greater than or equal to 3.9 is good enough, as + # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See + # https://docs.python.org/3/library/hashlib.html#:~:text=usedforsecurity for the version support notes. + if (sys_used.version_info.major > 3) or (sys_used.version_info.major == 3 and sys_used.version_info.minor >= 9): + return hash_function_object(usedforsecurity=False) + + # note that this can throw a ValueError in FIPS-enabled versions of Linux prior to 3.9 + # the OpenSSL hashlib will throw on first init here, but that is assumed to be responsibility of + # the caller to diagnose the ValueError & potentially display the error to screen. + return hash_function_object() + +def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): + """Checks if SCons has ability to call the default algorithms normally supported. + + This util class is sometimes called prior to setting the user-selected hash algorithm, + meaning that on FIPS-compliant systems the library would default-initialize MD5 + and throw an exception in set_hash_format. A common case is using the SConf options, + which can run prior to main, and thus ignore the options.hash_format variable. + + This function checks the DEFAULT_HASH_FORMATS and sets the ALLOWED_HASH_FORMATS + to only the ones that can be called. In Python >= 3.9 this will always default to + MD5 as in Python 3.9 there is an optional attribute "usedforsecurity" set for the method. + + Throws if no allowed hash formats are detected. + """ + global ALLOWED_HASH_FORMATS + _last_error = None + # note: if you call this method repeatedly, example using timeout, this is needed. + # otherwise it keeps appending valid formats to the string + ALLOWED_HASH_FORMATS = [] + + for test_algorithm in DEFAULT_HASH_FORMATS: + _test_hash = getattr(hashlib_used, test_algorithm, None) + # we know hashlib claims to support it... check to see if we can call it. + if _test_hash is not None: + # the hashing library will throw an exception on initialization in FIPS mode, + # meaning if we call the default algorithm returned with no parameters, it'll + # throw if it's a bad algorithm, otherwise it will append it to the known + # good formats. + try: + _attempt_init_of_python_3_9_hash_object(_test_hash, sys_used) + ALLOWED_HASH_FORMATS.append(test_algorithm) + except ValueError as e: + _last_error = e + continue + + if len(ALLOWED_HASH_FORMATS) == 0: + from SCons.Errors import SConsEnvironmentError # pylint: disable=import-outside-toplevel + # chain the exception thrown with the most recent error from hashlib. + raise SConsEnvironmentError( + 'No usable hash algorithms found.' + 'Most recent error from hashlib attached in trace.' + ) from _last_error + return + +_set_allowed_viable_default_hashes(hashlib) + def get_hash_format(): """Retrieves the hash format or ``None`` if not overridden. @@ -1684,8 +1760,27 @@ def get_hash_format(): """ return _HASH_FORMAT +def _attempt_get_hash_function(hash_name, hashlib_used=hashlib, sys_used=sys): + """Wrapper used to try to initialize a hash function given. + + If successful, returns the name of the hash function back to the user. + + Otherwise returns None. + """ + try: + _fetch_hash = getattr(hashlib_used, hash_name, None) + if _fetch_hash is None: + return None + _attempt_init_of_python_3_9_hash_object(_fetch_hash, sys_used) + return hash_name + except ValueError: + # if attempt_init_of_python_3_9 throws, this is typically due to FIPS being enabled + # however, if we get to this point, the viable hash function check has either been + # bypassed or otherwise failed to properly restrict the user to only the supported + # functions. As such throw the UserError as an internal assertion-like error. + return None -def set_hash_format(hash_format): +def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): """Sets the default hash format used by SCons. If `hash_format` is ``None`` or @@ -1702,24 +1797,58 @@ def set_hash_format(hash_format): if hash_format_lower not in ALLOWED_HASH_FORMATS: from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - raise UserError('Hash format "%s" is not supported by SCons. Only ' + # user can select something not supported by their OS but normally supported by + # SCons, example, selecting MD5 in an OS with FIPS-mode turned on. Therefore we first + # check if SCons supports it, and then if their local OS supports it. + if hash_format_lower in DEFAULT_HASH_FORMATS: + raise UserError('While hash format "%s" is supported by SCons, the ' + 'local system indicates only the following hash ' + 'formats are supported by the hashlib library: %s' % + (hash_format_lower, + ', '.join(ALLOWED_HASH_FORMATS)) + ) + else: + # the hash format isn't supported by SCons in any case. Warn the user, and + # if we detect that SCons supports more algorithms than their local system + # supports, warn the user about that too. + if ALLOWED_HASH_FORMATS == DEFAULT_HASH_FORMATS: + raise UserError('Hash format "%s" is not supported by SCons. Only ' 'the following hash formats are supported: %s' % (hash_format_lower, - ', '.join(ALLOWED_HASH_FORMATS))) - - _HASH_FUNCTION = getattr(hashlib, hash_format_lower, None) + ', '.join(ALLOWED_HASH_FORMATS)) + ) + else: + raise UserError('Hash format "%s" is not supported by SCons. ' + 'SCons supports more hash formats than your local system ' + 'is reporting; SCons supports: %s. Your local system only ' + 'supports: %s' % + (hash_format_lower, + ', '.join(DEFAULT_HASH_FORMATS), + ', '.join(ALLOWED_HASH_FORMATS)) + ) + + # this is not expected to fail. If this fails it means the set_allowed_viable_default_hashes + # function did not throw, or when it threw, the exception was caught and ignored, or + # the global ALLOWED_HASH_FORMATS was changed by an external user. + _HASH_FUNCTION = _attempt_get_hash_function(hash_format_lower, hashlib_used, sys_used) + if _HASH_FUNCTION is None: from SCons.Errors import UserError # pylint: disable=import-outside-toplevel raise UserError( - 'Hash format "%s" is not available in your Python interpreter.' + 'Hash format "%s" is not available in your Python interpreter. ' + 'Expected to be supported algorithm by set_allowed_viable_default_hashes, ' + 'Assertion error in SCons.' % hash_format_lower ) else: # Set the default hash format based on what is available, defaulting - # to md5 for backwards compatibility. + # to the first supported hash algorithm (usually md5) for backwards compatibility. + # in FIPS-compliant systems this usually defaults to SHA1, unless that too has been + # disabled. for choice in ALLOWED_HASH_FORMATS: - _HASH_FUNCTION = getattr(hashlib, choice, None) + _HASH_FUNCTION = _attempt_get_hash_function(choice, hashlib_used, sys_used) + if _HASH_FUNCTION is not None: break else: @@ -1728,7 +1857,9 @@ def set_hash_format(hash_format): raise UserError( 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' - 'SCons requires at least one.') + 'SCons requires at least one. Expected to support one or more ' + 'during set_allowed_viable_default_hashes.' + ) # Ensure that this is initialized in case either: # 1. This code is running in a unit test. @@ -1737,7 +1868,19 @@ def set_hash_format(hash_format): set_hash_format(None) -def _get_hash_object(hash_format): +def get_current_hash_algorithm_used(): + """Returns the current hash algorithm name used. + + Where the python version >= 3.9, this is expected to return md5. + If python's version is <= 3.8, this returns md5 on non-FIPS-mode platforms, and + sha1 or sha256 on FIPS-mode Linux platforms. + + This function is primarily useful for testing, where one expects a value to be + one of N distinct hashes, and therefore the test needs to know which hash to select. + """ + return _HASH_FUNCTION + +def _get_hash_object(hash_format, hashlib_used=hashlib, sys_used=sys): """Allocates a hash object using the requested hash format. Args: @@ -1752,7 +1895,7 @@ def _get_hash_object(hash_format): raise UserError('There is no default hash function. Did you call ' 'a hashing function before SCons was initialized?') - return _HASH_FUNCTION() + return _attempt_init_of_python_3_9_hash_object(getattr(hashlib_used, _HASH_FUNCTION, None), sys_used) if not hasattr(hashlib, hash_format): from SCons.Errors import UserError # pylint: disable=import-outside-toplevel @@ -1761,7 +1904,7 @@ def _get_hash_object(hash_format): 'Hash format "%s" is not available in your Python interpreter.' % hash_format) - return getattr(hashlib, hash_format)() + return _attempt_init_of_python_3_9_hash_object(getattr(hashlib, hash_format), sys_used) def hash_signature(s, hash_format=None): diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 616ea37..8001d2e 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -26,13 +26,17 @@ import io import os import sys import unittest -from collections import UserDict, UserList, UserString +import unittest.mock +import hashlib +import warnings +from collections import UserDict, UserList, UserString, namedtuple import TestCmd import SCons.Errors import SCons.compat from SCons.Util import ( + ALLOWED_HASH_FORMATS, AddPathIfNotExists, AppendPath, CLVar, @@ -42,6 +46,10 @@ from SCons.Util import ( Proxy, Selector, WhereIs, + _attempt_init_of_python_3_9_hash_object, + _attempt_get_hash_function, + _get_hash_object, + _set_allowed_viable_default_hashes, adjustixes, containsAll, containsAny, @@ -61,6 +69,7 @@ from SCons.Util import ( is_Tuple, print_tree, render_tree, + set_hash_format, silent_intern, splitext, to_String, @@ -838,12 +847,17 @@ class HashTestCase(unittest.TestCase): '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), }.items(): - hs = functools.partial(hash_signature, hash_format=algorithm) - s = list(map(hs, ('111', '222', '333'))) - - assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) - assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) - assert expected[2] == hash_collect(s, hash_format=algorithm) + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + hs = functools.partial(hash_signature, hash_format=algorithm) + s = list(map(hs, ('111', '222', '333'))) + + assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) + assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) + assert expected[2] == hash_collect(s, hash_format=algorithm) def test_MD5signature(self): """Test generating a signature""" @@ -855,11 +869,162 @@ class HashTestCase(unittest.TestCase): 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), }.items(): - s = hash_signature('111', hash_format=algorithm) - assert expected[0] == s, s - - s = hash_signature('222', hash_format=algorithm) - assert expected[1] == s, s + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + s = hash_signature('111', hash_format=algorithm) + assert expected[0] == s, s + + s = hash_signature('222', hash_format=algorithm) + assert expected[1] == s, s + +# this uses mocking out, which is platform specific, however, the FIPS +# behavior this is testing is also platform-specific, and only would be +# visible in hosts running Linux with the fips_mode kernel flag along +# with using OpenSSL. + +class FIPSHashTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(FIPSHashTestCase, self).__init__(*args, **kwargs) + + ############################### + # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 + self.fake_md5=lambda usedforsecurity=True: (usedforsecurity, 'md5') + self.fake_sha1=lambda usedforsecurity=True: (usedforsecurity, 'sha1') + self.fake_sha256=lambda usedforsecurity=True: (usedforsecurity, 'sha256') + ############################### + + ############################### + # hashlib mocks + md5Available = unittest.mock.Mock(md5=self.fake_md5) + del md5Available.sha1 + del md5Available.sha256 + self.md5Available=md5Available + + md5Default = unittest.mock.Mock(md5=self.fake_md5, sha1=self.fake_sha1) + del md5Default.sha256 + self.md5Default=md5Default + + sha1Default = unittest.mock.Mock(sha1=self.fake_sha1, sha256=self.fake_sha256) + del sha1Default.md5 + self.sha1Default=sha1Default + + sha256Default = unittest.mock.Mock(sha256=self.fake_sha256, **{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError}) + self.sha256Default=sha256Default + + all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) + self.all_throw=all_throw + + no_algorithms = unittest.mock.Mock() + del no_algorithms.md5 + del no_algorithms.sha1 + del no_algorithms.sha256 + del no_algorithms.nonexist + self.no_algorithms=no_algorithms + + unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) + del unsupported_algorithm.md5 + del unsupported_algorithm.sha1 + del unsupported_algorithm.sha256 + del unsupported_algorithm.unsupported + self.unsupported_algorithm=unsupported_algorithm + ############################### + + ############################### + # system version mocks + VersionInfo = namedtuple('VersionInfo', 'major minor micro releaselevel serial') + v3_8 = VersionInfo(3, 8, 199, 'super-beta', 1337) + v3_9 = VersionInfo(3, 9, 0, 'alpha', 0) + v4_8 = VersionInfo(4, 8, 0, 'final', 0) + + self.sys_v3_8 = unittest.mock.Mock(version_info=v3_8) + self.sys_v3_9 = unittest.mock.Mock(version_info=v3_9) + self.sys_v4_8 = unittest.mock.Mock(version_info=v4_8) + ############################### + + def test_basic_failover_bad_hashlib_hash_init(self): + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash init function returns None""" + assert _attempt_init_of_python_3_9_hash_object(None) is None + + def test_basic_failover_bad_hashlib_hash_get(self): + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash get function returns None""" + assert _attempt_get_hash_function("nonexist", self.no_algorithms) is None + + def test_usedforsecurity_flag_behavior(self): + """Test usedforsecurity flag -> should be set to 'True' on older versions of python, and 'False' on Python >= 3.9""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected + + def test_automatic_default_to_md5(self): + """Test automatic default to md5 even if sha1 available""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + _set_allowed_viable_default_hashes(self.md5Default, version) + set_hash_format(None, self.md5Default, version) + assert _get_hash_object(None, self.md5Default, version) == expected + + def test_automatic_default_to_sha256(self): + """Test automatic default to sha256 if other algorithms available but throw""" + for version, expected in { + self.sys_v3_8: (True, 'sha256'), + self.sys_v3_9: (False, 'sha256'), + self.sys_v4_8: (False, 'sha256'), + }.items(): + _set_allowed_viable_default_hashes(self.sha256Default, version) + set_hash_format(None, self.sha256Default, version) + assert _get_hash_object(None, self.sha256Default, version) == expected + + def test_automatic_default_to_sha1(self): + """Test automatic default to sha1 if md5 is missing from hashlib entirely""" + for version, expected in { + self.sys_v3_8: (True, 'sha1'), + self.sys_v3_9: (False, 'sha1'), + self.sys_v4_8: (False, 'sha1'), + }.items(): + _set_allowed_viable_default_hashes(self.sha1Default, version) + set_hash_format(None, self.sha1Default, version) + assert _get_hash_object(None, self.sha1Default, version) == expected + + def test_no_available_algorithms(self): + """expect exceptions on no available algorithms or when all algorithms throw""" + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.all_throw) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.unsupported_algorithm) + + def test_bad_algorithm_set_attempt(self): + """expect exceptions on user setting an unsupported algorithm selections, either by host or by SCons""" + + # nonexistant hash algorithm, not supported by SCons + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) + + # md5 is default-allowed, but in this case throws when we attempt to use it + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) + + def tearDown(self): + """Return SCons back to the normal global state for the hashing functions.""" + _set_allowed_viable_default_hashes(hashlib, sys) + set_hash_format(None) class NodeListTestCase(unittest.TestCase): diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 199e694..6b5bece 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1185,9 +1185,15 @@ to use the specified algorithm.</para> For example, <option>--hash-format=sha256</option> will create a SConsign database with name <filename>.sconsign_sha256.dblite</filename>.</para> -<para>If this option is not specified, a hash format of -md5 is used, and the SConsign database is -<filename>.sconsign.dblite</filename>.</para> +<para>If this option is not specified, a the first supported hash format found +is selected. Typically this is MD5, however, if you are on a FIPS-compliant system +and using a version of Python less than 3.9, SHA1 or SHA256 will be chosen as the default. +Python 3.9 and onwards clients will always default to MD5, even in FIPS mode, unless +otherwise specified with the <option>--hash-format</option> option.</para> + +<para>For MD5 databases (either explicitly specified with <option>--hash-format=md5</option> or +defaulted), the SConsign database is<filename>.sconsign.dblite</filename>. The newer SHA1 and +SHA256 selections meanwhile store their databases to <filename>.sconsign_algorithmname.dblite</filename></para> <para><emphasis>Available since &scons; 4.2.</emphasis></para> </listitem> diff --git a/test/Configure/ConfigureDryRunError.py b/test/Configure/ConfigureDryRunError.py index 3648518..224154b 100644 --- a/test/Configure/ConfigureDryRunError.py +++ b/test/Configure/ConfigureDryRunError.py @@ -31,6 +31,8 @@ import os import TestSCons +from SCons.Util import get_current_hash_algorithm_used + _obj = TestSCons._obj test = TestSCons.TestSCons() @@ -65,7 +67,16 @@ test.run(arguments='-n', status=2, stderr=expect) test.must_not_exist('config.log') test.subdir('.sconf_temp') -conftest_0_c = os.path.join(".sconf_temp", "conftest_df286a1d2f67e69d030b4eff75ca7e12_0.c") +# depending on which default hash function we're using, we'd expect one of the following filenames. +# The filenames are generated by the conftest changes in #3543 : https://github.com/SCons/scons/pull/3543/files +possible_filenames = { + 'md5': "conftest_df286a1d2f67e69d030b4eff75ca7e12_0.c", + 'sha1': "conftest_6e784ac3248d146c68396335df2f9428d78c1d24_0.c", + 'sha256': "conftest_be4b3c1d600e20dfc3e8d98748cd8193c850331643466d800ef8a4229a5be410_0.c" +} +test_filename = possible_filenames[get_current_hash_algorithm_used()] + +conftest_0_c = os.path.join(".sconf_temp", test_filename) SConstruct_file_line = test.python_file_line(SConstruct_path, 6)[:-1] expect = """ diff --git a/test/Configure/VariantDir-SConscript.py b/test/Configure/VariantDir-SConscript.py index deb7b8f..5818fc7 100644 --- a/test/Configure/VariantDir-SConscript.py +++ b/test/Configure/VariantDir-SConscript.py @@ -128,7 +128,7 @@ test.checkLogAndStdout( ["Checking for C header file math.h... ", import shutil shutil.rmtree(test.workpath(".sconf_temp")) -test.unlink(".sconsign.dblite") +test.unlink(test.get_sconsignname()+".dblite") # now with SConscriptChdir(1) test.run(arguments='chdir=yes') diff --git a/test/Configure/implicit-cache.py b/test/Configure/implicit-cache.py index f4f3e94..2cf74d1 100644 --- a/test/Configure/implicit-cache.py +++ b/test/Configure/implicit-cache.py @@ -55,6 +55,7 @@ get longer and longer until it blew out the users's memory. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSConsign +from SCons.Util import get_hash_format, get_current_hash_algorithm_used test = TestSConsign.TestSConsign() @@ -75,7 +76,28 @@ test.write('foo.h', "#define FOO 1\n") test.run(arguments = '.') -test.run_sconsign('-d .sconf_temp -e conftest_5a3fa36d51dd2a28d521d6cc0e2e1d04_0.c --raw .sconsign.dblite') +# depending on which default hash function we're using, we'd expect one of the following filenames. +# The filenames are generated by the conftest changes in #3543 : https://github.com/SCons/scons/pull/3543/files +# this test is different than the other tests, as the database name is used here. +# when the database defaults to md5, that's a different name than when the user selects md5 directly. +possible_filenames = { + 'default': "conftest_5a3fa36d51dd2a28d521d6cc0e2e1d04_0.c", + 'md5': "conftest_5a3fa36d51dd2a28d521d6cc0e2e1d04_0.c", + 'sha1': "conftest_80e5b88f2c7427a92f0e6c7184f144f874f10e60_0.c", + 'sha256': "conftest_ba8270c26647ad00993cd7777f4c5d3751018372b97d16eb993563bea051c3df_0.c" +} +# user left algorithm default, it defaulted to md5, with the special database name +if get_hash_format() is None and get_current_hash_algorithm_used() == 'md5': + test_filename = possible_filenames['default'] +# either user selected something (like explicitly setting md5) or algorithm defaulted to something else. +# SCons can default to something else if it detects the hashlib doesn't support it, example md5 in FIPS +# mode prior to Python 3.9 +else: + test_filename = possible_filenames[get_current_hash_algorithm_used()] + +database_name=test.get_sconsignname() + ".dblite" + +test.run_sconsign('-d .sconf_temp -e {} --raw {}'.format(test_filename, database_name)) old_sconsign_dblite = test.stdout() # Second run: Have the configure subsystem also look for foo.h, so @@ -88,11 +110,11 @@ old_sconsign_dblite = test.stdout() test.run(arguments = '--implicit-cache USE_FOO=1 .') -test.run_sconsign('-d .sconf_temp -e conftest_5a3fa36d51dd2a28d521d6cc0e2e1d04_0.c --raw .sconsign.dblite') +test.run_sconsign('-d .sconf_temp -e {} --raw {}'.format(test_filename, database_name)) new_sconsign_dblite = test.stdout() if old_sconsign_dblite != new_sconsign_dblite: - print(".sconsign.dblite did not match:") + print("{} did not match:".format(database_name)) print("FIRST RUN ==========") print(old_sconsign_dblite) print("SECOND RUN ==========") diff --git a/test/Configure/option--config.py b/test/Configure/option--config.py index f31336f..3fd0691 100644 --- a/test/Configure/option--config.py +++ b/test/Configure/option--config.py @@ -32,6 +32,7 @@ import os.path from TestSCons import TestSCons, ConfigCheckInfo, _obj from TestCmd import IS_WINDOWS +from SCons.Util import get_current_hash_algorithm_used test = TestSCons() @@ -42,6 +43,11 @@ CR = test.CR # cached rebuild (up to date) NCF = test.NCF # non-cached build failure CF = test.CF # cached build failure +# as this test is somewhat complicated, skip it if the library doesn't support md5 +# as the default hashing algorithm. +if get_current_hash_algorithm_used() != 'md5': + test.skip_test('Skipping test as could not continue without the hash algorithm set to md5!') + SConstruct_path = test.workpath('SConstruct') test.write(SConstruct_path, """ diff --git a/test/Removed/SourceSignatures/Old/no-csigs.py b/test/Removed/SourceSignatures/Old/no-csigs.py index 60c0460..483eeea 100644 --- a/test/Removed/SourceSignatures/Old/no-csigs.py +++ b/test/Removed/SourceSignatures/Old/no-csigs.py @@ -66,7 +66,7 @@ f2.out: \S+ \d+ \d+ \S+ \[build\(target, source, env\)\] """ -test.run_sconsign(arguments = test.workpath('.sconsign'), +test.run_sconsign(arguments = test.workpath(test.get_sconsignname()), stdout = expect) diff --git a/test/Repository/variants.py b/test/Repository/variants.py index f89605b..c95e853 100644 --- a/test/Repository/variants.py +++ b/test/Repository/variants.py @@ -216,12 +216,14 @@ repository/src1/bbb.c: REPOSITORY_FOO repository/src1/main.c: REPOSITORY_FOO """) +database_name=test.get_sconsignname() + test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) test.run(program=repository_build2_foo_src2_xxx_xxx, stdout="""\ repository/src2/include/my_string.h: FOO @@ -236,11 +238,11 @@ repository/src2/xxx/main.c: BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) # Make the entire repository non-writable, so we'll detect # if we try to write into it accidentally. @@ -270,11 +272,11 @@ repository/src1/main.c: REPOSITORY_BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) test.up_to_date(chdir='work1', options=opts + " OS=bar", arguments='build1') @@ -301,11 +303,11 @@ repository/src1/main.c: WORK_BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) test.up_to_date(chdir='work1', options=opts + " OS=bar", arguments='build1') @@ -319,11 +321,11 @@ repository/src1/main.c: WORK_FOO """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) test.up_to_date(chdir='work1', options=opts + " OS=foo", arguments='build1') @@ -376,11 +378,11 @@ repository/src2/xxx/main.c: BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) # Ensure file time stamps will be newer. time.sleep(2) @@ -411,11 +413,11 @@ repository/src2/xxx/main.c: BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) # test.unlink(['work2', 'src2', 'include', 'my_string.h']) @@ -435,11 +437,11 @@ repository/src2/xxx/main.c: BAR """) test.fail_test(os.path.exists( - test.workpath('repository', 'src1', '.sconsign'))) + test.workpath('repository', 'src1', database_name))) test.fail_test(os.path.exists( - test.workpath('repository', 'src2', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work1', 'src1', '.sconsign'))) -test.fail_test(os.path.exists(test.workpath('work2', 'src2', '.sconsign'))) + test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) # test.pass_test() diff --git a/test/SConsignFile/default.py b/test/SConsignFile/default.py index 868f9d7..99ae54e 100644 --- a/test/SConsignFile/default.py +++ b/test/SConsignFile/default.py @@ -59,9 +59,12 @@ test.write(['subdir', 'f4.in'], "subdir/f4.in\n") test.run() -test.must_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +database_name = test.get_sconsignname() +database_filename = database_name + ".dblite" + +test.must_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -70,9 +73,9 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments='.') -test.must_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +test.must_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.pass_test() diff --git a/test/SConsignFile/explicit-dbm-module.py b/test/SConsignFile/explicit-dbm-module.py index c093271..628d8b5 100644 --- a/test/SConsignFile/explicit-dbm-module.py +++ b/test/SConsignFile/explicit-dbm-module.py @@ -61,9 +61,12 @@ test.write(['subdir', 'f4.in'], "subdir/f4.in\n") test.run() -test.must_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +database_name = test.get_sconsignname() +database_filename = database_name + ".dblite" + +test.must_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -72,9 +75,9 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments='.') -test.must_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +test.must_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.pass_test() diff --git a/test/SConsignFile/explicit-file.py b/test/SConsignFile/explicit-file.py index 850b0ef..c601f7f 100644 --- a/test/SConsignFile/explicit-file.py +++ b/test/SConsignFile/explicit-file.py @@ -59,9 +59,10 @@ test.write(['subdir', 'f8.in'], "subdir/f8.in\n") test.run() +database_name = test.get_sconsignname() test.must_exist(test.workpath('my_sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.must_match('f5.out', "f5.in\n") test.must_match('f6.out', "f6.in\n") @@ -71,8 +72,8 @@ test.must_match(['subdir', 'f8.out'], "subdir/f8.in\n") test.up_to_date(arguments='.') test.must_exist(test.workpath('my_sconsign.dblite')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) +test.must_not_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath('subdir', database_name)) test.pass_test() diff --git a/test/SConsignFile/make-directory.py b/test/SConsignFile/make-directory.py index 264ee26..f2a0db1 100644 --- a/test/SConsignFile/make-directory.py +++ b/test/SConsignFile/make-directory.py @@ -57,7 +57,8 @@ test.must_not_exist(['bar', 'foo.txt']) test.must_not_exist('sub') test.must_not_exist(['sub', 'dir']) -test.must_not_exist(['sub', 'dir', '.sconsign.dblite']) +database_name = test.get_sconsignname() +test.must_not_exist(['sub', 'dir', database_name + '.dblite']) test.run(stdout=expect) diff --git a/test/SConsignFile/use-dbhash.py b/test/SConsignFile/use-dbhash.py index 2968cd7..65eb92c 100644 --- a/test/SConsignFile/use-dbhash.py +++ b/test/SConsignFile/use-dbhash.py @@ -48,10 +48,12 @@ with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: sys.exit(0) """) +database_name = test.get_sconsignname() + # test.write('SConstruct', """ import %(use_dbm)s -SConsignFile('.sconsign', %(use_dbm)s) +SConsignFile('%(database_name)s', %(use_dbm)s) DefaultEnvironment(tools=[]) B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS={'B': B}, tools=[]) @@ -68,10 +70,12 @@ test.write(['subdir', 'f4.in'], "subdir/f4.in\n") test.run() -test.must_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +database_name = test.get_sconsignname() +database_filename = database_name + ".dblite" +test.must_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -80,10 +84,10 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments = '.') -test.must_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +test.must_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.pass_test() diff --git a/test/SConsignFile/use-dbm.py b/test/SConsignFile/use-dbm.py index a1ef1b2..6b6c96b 100644 --- a/test/SConsignFile/use-dbm.py +++ b/test/SConsignFile/use-dbm.py @@ -49,10 +49,11 @@ with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: sys.exit(0) """) +database_name = test.get_sconsignname() # test.write('SConstruct', """ import %(use_dbm)s -SConsignFile('.sconsign', %(use_dbm)s) +SConsignFile('%(database_name)s', %(use_dbm)s) DefaultEnvironment(tools=[]) B = Builder(action=r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS={'B': B}, tools=[]) @@ -72,13 +73,15 @@ test.run() # We don't check for explicit .db or other file, because base "dbm" # can use different file extensions on different implementations. +database_name = test.get_sconsignname() +database_filename = database_name + '.dblite' test.fail_test( - os.path.exists('.sconsign') and 'dbm' not in dbm.whichdb('.sconsign'), - message=".sconsign existed and wasn't any type of dbm file", + os.path.exists(database_name) and 'dbm' not in dbm.whichdb(database_name), + message="{} existed and wasn't any type of dbm file".format(database_name), ) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -87,11 +90,11 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments='.') -test.fail_test(os.path.exists('.sconsign') and 'dbm' not in dbm.whichdb('.sconsign'), - message=".sconsign existed and wasn't any type of dbm file") -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +test.fail_test(os.path.exists(database_name) and 'dbm' not in dbm.whichdb(database_name), + message="{} existed and wasn't any type of dbm file".format(database_name)) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.pass_test() diff --git a/test/SConsignFile/use-dumbdbm.py b/test/SConsignFile/use-dumbdbm.py index 875f3fc..e7fb091 100644 --- a/test/SConsignFile/use-dumbdbm.py +++ b/test/SConsignFile/use-dumbdbm.py @@ -48,10 +48,11 @@ with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: sys.exit(0) """) +database_name = test.get_sconsignname() # test.write('SConstruct', """ import %(use_dbm)s -SConsignFile('.sconsign', %(use_dbm)s) +SConsignFile('%(database_name)s', %(use_dbm)s) DefaultEnvironment(tools=[]) B = Builder(action=r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS={'B': B}, tools=[]) @@ -68,14 +69,14 @@ test.write(['subdir', 'f4.in'], "subdir/f4.in\n") test.run() -test.must_exist(test.workpath('.sconsign.dat')) -test.must_exist(test.workpath('.sconsign.dir')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dat')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dir')) +test.must_exist(test.workpath('{}.dat'.format(database_name))) +test.must_exist(test.workpath('{}.dir'.format(database_name))) +test.must_not_exist(test.workpath('{}'.format(database_name))) +test.must_not_exist(test.workpath('{}.dblite'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dblite'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dat'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dir'.format(database_name))) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -84,14 +85,14 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments='.') -test.must_exist(test.workpath('.sconsign.dat')) -test.must_exist(test.workpath('.sconsign.dir')) -test.must_not_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dat')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dir')) +test.must_exist(test.workpath('{}.dat'.format(database_name))) +test.must_exist(test.workpath('{}.dir'.format(database_name))) +test.must_not_exist(test.workpath('{}'.format(database_name))) +test.must_not_exist(test.workpath('{}.dblite'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dblite'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dat'.format(database_name))) +test.must_not_exist(test.workpath('subdir', '{}.dir'.format(database_name))) test.pass_test() diff --git a/test/SConsignFile/use-gdbm.py b/test/SConsignFile/use-gdbm.py index c1f0c4d..11ae052 100644 --- a/test/SConsignFile/use-gdbm.py +++ b/test/SConsignFile/use-gdbm.py @@ -48,10 +48,12 @@ with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: sys.exit(0) """) +database_name = test.get_sconsignname() +database_filename = database_name + '.dblite' # test.write('SConstruct', """ import %(use_dbm)s -SConsignFile('.sconsign', %(use_dbm)s) +SConsignFile('%(database_name)s', %(use_dbm)s) DefaultEnvironment(tools=[]) B = Builder(action='%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS={'B': B}, tools=[]) @@ -68,10 +70,10 @@ test.write(['subdir', 'f4.in'], "subdir/f4.in\n") test.run() -test.must_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +test.must_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.must_match('f1.out', "f1.in\n") test.must_match('f2.out', "f2.in\n") @@ -80,10 +82,10 @@ test.must_match(['subdir', 'f4.out'], "subdir/f4.in\n") test.up_to_date(arguments='.') -test.must_exist(test.workpath('.sconsign')) -test.must_not_exist(test.workpath('.sconsign.dblite')) -test.must_not_exist(test.workpath('subdir', '.sconsign')) -test.must_not_exist(test.workpath('subdir', '.sconsign.dblite')) +test.must_exist(test.workpath(database_name)) +test.must_not_exist(test.workpath(database_filename)) +test.must_not_exist(test.workpath('subdir', database_name)) +test.must_not_exist(test.workpath('subdir', database_filename)) test.pass_test() diff --git a/test/implicit/changed-node.py b/test/implicit/changed-node.py index c8c5a01..d89c14b 100644 --- a/test/implicit/changed-node.py +++ b/test/implicit/changed-node.py @@ -126,8 +126,9 @@ test.pass_test() #def clean(full=0): # for f in ('d','b','a','SConstruct'): # rm(f) +# database_name = test.get_sconsignname() # if full: -# for f in ('.sconsign.dblite', 'build.py'): +# for f in (database_name + '.dblite', 'build.py'): # rm(f) # #clean(1) diff --git a/test/option/hash-format.py b/test/option/hash-format.py index 9fa10ee..663ad2e 100644 --- a/test/option/hash-format.py +++ b/test/option/hash-format.py @@ -27,28 +27,51 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import hashlib import os import TestSCons +from SCons.Util import ALLOWED_HASH_FORMATS, DEFAULT_HASH_FORMATS # Test passing the hash format by command-line. INVALID_ALGORITHM = 'testfailure' -for algorithm in ['md5', 'sha1', 'sha256', INVALID_ALGORITHM, None]: + +for algorithm in [*DEFAULT_HASH_FORMATS, INVALID_ALGORITHM, None]: test = TestSCons.TestSCons() test.dir_fixture('hash-format') + # Expect failure due to an unsupported/invalid algorithm. + # The error message however changes if SCons detects that the host system doesn't support one or more algorithms + # Primary reason the message changes is so user doesn't have to start with unsupported algorithm A and then attempt + # to switch to unsupported algorithm B. + # On normal systems (allowed=default) this will output a fixed message, but on FIPS-enabled or other weird systems + # that don't have default allowed algorithms, it informs the user of the mismatch _and_ the currently supported + # algorithms on the system they're using. + # In Python 3.9 this becomes somewhat obselete as the hashlib is informed we don't use hashing for security but + # for loose integrity. if algorithm == INVALID_ALGORITHM: - # Expect failure due to an unsupported/invalid algorithm. - test.run('--hash-format=%s .' % algorithm, stderr=r""" -scons: \*\*\* Hash format "{}" is not supported by SCons. Only the following hash formats are supported: md5, sha1, sha256 + if ALLOWED_HASH_FORMATS == DEFAULT_HASH_FORMATS: + test.run('--hash-format=%s .' % algorithm, stderr=r""" +scons: \*\*\* Hash format "{}" is not supported by SCons. Only the following hash formats are supported: {} +File "[^"]+", line \d+, in \S+ +""".format(algorithm, ', '.join(DEFAULT_HASH_FORMATS)), status=2, match=TestSCons.match_re) + else: + test.run('--hash-format=%s .' % algorithm, stderr=r""" +scons: \*\*\* Hash format "{}" is not supported by SCons. SCons supports more hash formats than your local system is reporting; SCons supports: {}. Your local system only supports: {} File "[^"]+", line \d+, in \S+ -""".format(algorithm), status=2, match=TestSCons.match_re) +""".format(algorithm, ', '.join(DEFAULT_HASH_FORMATS), ', '.join(ALLOWED_HASH_FORMATS)), status=2, match=TestSCons.match_re) continue elif algorithm is not None: - # Skip any algorithm that the Python interpreter doesn't have. - if hasattr(hashlib, algorithm): + if algorithm in ALLOWED_HASH_FORMATS: expected_dblite = test.workpath('.sconsign_%s.dblite' % algorithm) test.run('--hash-format=%s .' % algorithm) else: - print('Skipping test with --hash-format=%s because that ' - 'algorithm is not available.' % algorithm) + test.run('--hash-format=%s' % algorithm, stderr=r""" +scons: \*\*\* While hash format "{}" is supported by SCons, the local system indicates only the following hash formats are supported by the hashlib library: {} +File "[^"]+", line \d+, in \S+ +Error in atexit._run_exitfuncs: +Traceback \(most recent call last\): + File "[^"]+", line \d+, in \S+ + assert csig == '[a-z0-9]+', csig +AssertionError: [a-z0-9]+ +""".format(algorithm, ', '.join(ALLOWED_HASH_FORMATS)), status=2, match=TestSCons.match_re) + continue else: # The SConsign file in the hash-format folder has logic to call # SCons.Util.set_hash_format('sha256') if the default algorithm is diff --git a/test/option/option-n.py b/test/option/option-n.py index e647b8e..eb8eb62 100644 --- a/test/option/option-n.py +++ b/test/option/option-n.py @@ -43,6 +43,7 @@ import os import re import TestSCons +from SCons.Util import get_current_hash_algorithm_used _python_ = TestSCons._python_ @@ -118,7 +119,7 @@ test.fail_test(not os.path.exists(test.workpath('f1.out'))) expect = test.wrap_stdout("""\ %(_python_)s build.py f1.out """ % locals()) -test.unlink('.sconsign.dblite') +test.unlink(test.get_sconsignname()+'.dblite') test.write('f1.out', "X1.out\n") test.run(arguments='-n f1.out', stdout=expect) test.run(arguments='-n f1.out', stdout=expect) @@ -204,13 +205,23 @@ test.run(arguments="-n", stderr=stderr, status=2, test.fail_test(os.path.exists(test.workpath("configure", "config.test"))) test.fail_test(os.path.exists(test.workpath("configure", "config.log"))) + +# depending on which default hash function we're using, we'd expect one of the following filenames. +# The filenames are generated by the conftest changes in #3543 : https://github.com/SCons/scons/pull/3543/files +possible_filenames = { + 'md5': "conftest_b10a8db164e0754105b7a99be72e3fe5_0.in", + 'sha1': "conftest_0a4d55a8d778e5022fab701977c5d840bbc486d0_0.in", + 'sha256': "conftest_a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e_0.in" +} +test_filename = possible_filenames[get_current_hash_algorithm_used()] + # test that targets are not built, if conf_dir exists. # verify that .cache and config.log are not created. # an error should be raised stderr = r""" scons: \*\*\* Cannot update configure test "%s" within a dry-run\. File \S+, line \S+, in \S+ -""" % re.escape(os.path.join("config.test", "conftest_b10a8db164e0754105b7a99be72e3fe5_0.in")) +""" % re.escape(os.path.join("config.test", test_filename)) test.subdir(['configure', 'config.test']) test.run(arguments="-n", stderr=stderr, status=2, chdir=test.workpath("configure")) diff --git a/test/question/Configure.py b/test/question/Configure.py index 7df29f5..d01d0fa 100644 --- a/test/question/Configure.py +++ b/test/question/Configure.py @@ -36,6 +36,7 @@ import re import TestCmd import TestSCons +from SCons.Util import get_current_hash_algorithm_used test = TestSCons.TestSCons(match = TestCmd.match_re_dotall) @@ -80,13 +81,22 @@ test.run(arguments="-q aaa.out",stderr=stderr,status=2) test.must_not_exist(test.workpath("config.test")) test.must_not_exist(test.workpath("config.log")) +# depending on which default hash function we're using, we'd expect one of the following filenames. +# The filenames are generated by the conftest changes in #3543 : https://github.com/SCons/scons/pull/3543/files +possible_filenames = { + 'md5': "conftest_b10a8db164e0754105b7a99be72e3fe5_0.in", + 'sha1': "conftest_0a4d55a8d778e5022fab701977c5d840bbc486d0_0.in", + 'sha256': "conftest_a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e_0.in" +} +test_filename = possible_filenames[get_current_hash_algorithm_used()] + # test that targets are not built, if conf_dir exists. # verify that .cache and config.log are not created. # an error should be raised stderr=r""" scons: \*\*\* Cannot update configure test "%s" within a dry-run\. File \S+, line \S+, in \S+ -""" % re.escape(os.path.join("config.test", "conftest_b10a8db164e0754105b7a99be72e3fe5_0.in")) +""" % re.escape(os.path.join("config.test", test_filename)) test.subdir('config.test') @@ -94,7 +104,7 @@ test.run(arguments="-q aaa.out",stderr=stderr,status=2) test.must_not_exist(test.workpath("config.test", ".cache")) test.must_not_exist(test.workpath("config.test", "conftest_0")) -test.must_not_exist(test.workpath("config.test", "conftest_b10a8db164e0754105b7a99be72e3fe5_0.in")) +test.must_not_exist(test.workpath("config.test", test_filename)) test.must_not_exist(test.workpath("config.log")) # test that no error is raised, if all targets are up-to-date. In this diff --git a/test/sconsign/corrupt.py b/test/sconsign/corrupt.py index 25b48e2..fa6a0e9 100644 --- a/test/sconsign/corrupt.py +++ b/test/sconsign/corrupt.py @@ -30,14 +30,20 @@ Test that we get proper warnings when .sconsign* files are corrupt. import TestSCons import TestCmd +import re test = TestSCons.TestSCons(match = TestCmd.match_re) test.subdir('work1', ['work1', 'sub'], 'work2', ['work2', 'sub']) -work1__sconsign_dblite = test.workpath('work1', '.sconsign.dblite') -work2_sub__sconsign = test.workpath('work2', 'sub', '.sconsign') +database_name = test.get_sconsignname() +database_filename = database_name + ".dblite" + +# for test1 we're using the default database filename +work1__sconsign_dblite = test.workpath('work1', database_filename) +# for test 2 we have an explicit hardcode to .sconsign +work2_sub__sconsign = test.workpath('work2', 'sub', database_name) SConstruct_contents = """\ def build1(target, source, env): @@ -57,9 +63,9 @@ test.write(['work1', 'SConstruct'], SConstruct_contents) test.write(['work1', 'foo.in'], "work1/foo.in\n") stderr = r''' -scons: warning: Ignoring corrupt .sconsign file: \.sconsign\.dblite +scons: warning: Ignoring corrupt .sconsign file: {} .* -''' +'''.format(re.escape(database_filename)) stdout = test.wrap_stdout(r'build1\(\["sub.foo\.out"\], \["foo\.in"\]\)' + '\n') @@ -82,9 +88,9 @@ test.write(['work2', 'SConstruct'], SConstruct_contents) test.write(['work2', 'foo.in'], "work2/foo.in\n") stderr = r''' -scons: warning: Ignoring corrupt .sconsign file: sub.\.sconsign +scons: warning: Ignoring corrupt .sconsign file: sub.{} .* -''' +'''.format(database_name) stdout = test.wrap_stdout(r'build1\(\["sub.foo\.out"\], \["foo\.in"\]\)' + '\n') diff --git a/test/sconsign/nonwritable.py b/test/sconsign/nonwritable.py index 812a476..e952078 100644 --- a/test/sconsign/nonwritable.py +++ b/test/sconsign/nonwritable.py @@ -44,10 +44,11 @@ test.subdir('work1', ['work2', 'sub2'], ['work2', 'sub3']) -work1__sconsign_dblite = test.workpath('work1', '.sconsign.dblite') -work2_sub1__sconsign = test.workpath('work2', 'sub1', '.sconsign') -work2_sub2__sconsign = test.workpath('work2', 'sub2', '.sconsign') -work2_sub3__sconsign = test.workpath('work2', 'sub3', '.sconsign') +database_name = test.get_sconsignname() +work1__sconsign_dblite = test.workpath('work1', database_name + '.dblite') +work2_sub1__sconsign = test.workpath('work2', 'sub1', database_name) +work2_sub2__sconsign = test.workpath('work2', 'sub2', database_name) +work2_sub3__sconsign = test.workpath('work2', 'sub3', database_name) SConstruct_contents = """\ def build1(target, source, env): diff --git a/test/sconsign/script/Configure.py b/test/sconsign/script/Configure.py index 125022d..02a2c20 100644 --- a/test/sconsign/script/Configure.py +++ b/test/sconsign/script/Configure.py @@ -66,7 +66,7 @@ env = conf.Finish() test.run(arguments = '.') -sig_re = r'[0-9a-fA-F]{32}' +sig_re = r'[0-9a-fA-F]{32,64}' date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d' _sconf_temp_conftest_0_c = '.sconf_temp/conftest_%(sig_re)s_0.c'%locals() @@ -90,7 +90,9 @@ conftest_%(sig_re)s_0_%(sig_re)s%(_obj)s: %(CC_file)s: %(sig_re)s \d+ \d+ """ % locals() -test.run_sconsign(arguments = ".sconsign", +# grab .sconsign or .sconsign_<hashname> +database_name=test.get_sconsignname() +test.run_sconsign(arguments = database_name, stdout = expect) test.pass_test() diff --git a/test/sconsign/script/SConsignFile.py b/test/sconsign/script/SConsignFile.py index c54e17b..4e29ef0 100644 --- a/test/sconsign/script/SConsignFile.py +++ b/test/sconsign/script/SConsignFile.py @@ -141,9 +141,11 @@ test.write(['sub2', 'inc2.h'], r"""\ test.run(arguments = '--implicit-cache .') -sig_re = r'[0-9a-fA-F]{32}' +sig_re = r'[0-9a-fA-F]{32,64}' -test.run_sconsign(arguments = ".sconsign", +database_name = test.get_sconsignname() + +test.run_sconsign(arguments = database_name, stdout = r"""=== .: SConstruct: None \d+ \d+ fake_cc\.py: %(sig_re)s \d+ \d+ @@ -174,7 +176,7 @@ inc1.h: %(sig_re)s \d+ \d+ inc2.h: %(sig_re)s \d+ \d+ """ % locals()) -test.run_sconsign(arguments = "--raw .sconsign", +test.run_sconsign(arguments = "--raw " + database_name, stdout = r"""=== .: SConstruct: {'csig': None, 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2} fake_cc\.py: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2} @@ -302,9 +304,9 @@ inc2.h: size: \d+ """ % locals() -test.run_sconsign(arguments = "-v .sconsign", stdout=expect) +test.run_sconsign(arguments = "-v " + database_name, stdout=expect) -test.run_sconsign(arguments = "-c -v .sconsign", +test.run_sconsign(arguments = "-c -v " + database_name, stdout = r"""=== .: SConstruct: csig: None @@ -332,7 +334,7 @@ inc2.h: csig: %(sig_re)s """ % locals()) -test.run_sconsign(arguments = "-s -v .sconsign", +test.run_sconsign(arguments = "-s -v " + database_name, stdout = r"""=== .: SConstruct: size: \d+ @@ -360,7 +362,7 @@ inc2.h: size: \d+ """ % locals()) -test.run_sconsign(arguments = "-t -v .sconsign", +test.run_sconsign(arguments = "-t -v " + database_name, stdout = r"""=== .: SConstruct: timestamp: \d+ @@ -388,7 +390,7 @@ inc2.h: timestamp: \d+ """ % locals()) -test.run_sconsign(arguments = "-e hello.obj .sconsign", +test.run_sconsign(arguments = "-e hello.obj " + database_name, stdout = r"""=== .: === sub1: hello.obj: %(sig_re)s \d+ \d+ @@ -406,7 +408,7 @@ hello.obj: %(sig_re)s \d+ \d+ stderr = r"""sconsign: no entry `hello\.obj' in `\.' """ % locals()) -test.run_sconsign(arguments = "-e hello.obj -e hello.exe -e hello.obj .sconsign", +test.run_sconsign(arguments = "-e hello.obj -e hello.exe -e hello.obj " + database_name, stdout = r"""=== .: === sub1: hello.obj: %(sig_re)s \d+ \d+ @@ -444,7 +446,7 @@ sconsign: no entry `hello\.exe' in `\.' sconsign: no entry `hello\.obj' in `\.' """ % locals()) -#test.run_sconsign(arguments = "-i -v .sconsign", +#test.run_sconsign(arguments = "-i -v " + database_name, # stdout = r"""=== sub1: #hello.exe: # implicit: diff --git a/test/sconsign/script/Signatures.py b/test/sconsign/script/Signatures.py index 0f21496..ced5b44 100644 --- a/test/sconsign/script/Signatures.py +++ b/test/sconsign/script/Signatures.py @@ -149,10 +149,12 @@ test.sleep() test.run(arguments = '. --max-drift=1') -sig_re = r'[0-9a-fA-F]{32}' +sig_re = r'[0-9a-fA-F]{32,64}' date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d' -test.run_sconsign(arguments = "-e hello.exe -e hello.obj sub1/.sconsign", +database_name = test.get_sconsignname() + +test.run_sconsign(arguments = "-e hello.exe -e hello.obj sub1/{}".format(database_name), stdout = r"""hello.exe: %(sig_re)s \d+ \d+ %(sub1_hello_obj)s: %(sig_re)s \d+ \d+ fake_link\.py: None \d+ \d+ @@ -163,7 +165,7 @@ hello.obj: %(sig_re)s \d+ \d+ %(sig_re)s \[.*\] """ % locals()) -test.run_sconsign(arguments = "-e hello.exe -e hello.obj -r sub1/.sconsign", +test.run_sconsign(arguments = "-e hello.exe -e hello.obj -r sub1/{}".format(database_name), stdout = r"""hello.exe: %(sig_re)s '%(date_re)s' \d+ %(sub1_hello_obj)s: %(sig_re)s '%(date_re)s' \d+ fake_link\.py: None '%(date_re)s' \d+ diff --git a/test/sconsign/script/dblite.py b/test/sconsign/script/dblite.py index 41cf7e2..1fcf8c0 100644 --- a/test/sconsign/script/dblite.py +++ b/test/sconsign/script/dblite.py @@ -109,7 +109,7 @@ test.sleep() test.run(arguments = '. --max-drift=1') -sig_re = r'[0-9a-fA-F]{32}' +sig_re = r'[0-9a-fA-F]{32,64}' date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d' if sys.platform == 'win32': diff --git a/test/sconsign/script/no-SConsignFile.py b/test/sconsign/script/no-SConsignFile.py index c345829..bdd878b 100644 --- a/test/sconsign/script/no-SConsignFile.py +++ b/test/sconsign/script/no-SConsignFile.py @@ -40,6 +40,8 @@ test = TestSConsign.TestSConsign(match = TestSConsign.match_re) test.subdir('sub1', 'sub2') +database_name = test.get_sconsignname() + # Because this test sets SConsignFile(None), we execute our fake # scripts directly, not by feeding them to the Python executable. # That is, we chmod 0o755 and use a "#!/usr/bin/env python" first @@ -152,7 +154,7 @@ test.write(['sub2', 'inc2.h'], r"""\ test.run(arguments = '--implicit-cache --tree=prune .') -sig_re = r'[0-9a-fA-F]{32}' +sig_re = r'[0-9a-fA-F]{32,64}' expect = r"""hello.c: %(sig_re)s \d+ \d+ hello.exe: %(sig_re)s \d+ \d+ @@ -165,9 +167,9 @@ hello.obj: %(sig_re)s \d+ \d+ %(sig_re)s \[.*\] """ % locals() -test.run_sconsign(arguments = "sub1/.sconsign", stdout=expect) +test.run_sconsign(arguments = "sub1/{}".format(database_name), stdout=expect) -test.run_sconsign(arguments = "--raw sub1/.sconsign", +test.run_sconsign(arguments = "--raw sub1/{}".format(database_name), stdout = r"""hello.c: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2} hello.exe: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2} %(sub1_hello_obj)s: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2} @@ -179,7 +181,7 @@ hello.obj: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_i %(sig_re)s \[.*\] """ % locals()) -test.run_sconsign(arguments = "-v sub1/.sconsign", +test.run_sconsign(arguments = "-v sub1/{}".format(database_name), stdout = r"""hello.c: csig: %(sig_re)s timestamp: \d+ @@ -214,7 +216,7 @@ hello.obj: action: %(sig_re)s \[.*\] """ % locals()) -test.run_sconsign(arguments = "-c -v sub1/.sconsign", +test.run_sconsign(arguments = "-c -v sub1/{}".format(database_name), stdout = r"""hello.c: csig: %(sig_re)s hello.exe: @@ -223,7 +225,7 @@ hello.obj: csig: %(sig_re)s """ % locals()) -test.run_sconsign(arguments = "-s -v sub1/.sconsign", +test.run_sconsign(arguments = "-s -v sub1/{}".format(database_name), stdout = r"""hello.c: size: \d+ hello.exe: @@ -232,7 +234,7 @@ hello.obj: size: \d+ """ % locals()) -test.run_sconsign(arguments = "-t -v sub1/.sconsign", +test.run_sconsign(arguments = "-t -v sub1/{}".format(database_name), stdout = r"""hello.c: timestamp: \d+ hello.exe: @@ -241,14 +243,14 @@ hello.obj: timestamp: \d+ """ % locals()) -test.run_sconsign(arguments = "-e hello.obj sub1/.sconsign", +test.run_sconsign(arguments = "-e hello.obj sub1/{}".format(database_name), stdout = r"""hello.obj: %(sig_re)s \d+ \d+ %(sub1_hello_c)s: %(sig_re)s \d+ \d+ fake_cc\.py: %(sig_re)s \d+ \d+ %(sig_re)s \[.*\] """ % locals()) -test.run_sconsign(arguments = "-e hello.obj -e hello.exe -e hello.obj sub1/.sconsign", +test.run_sconsign(arguments = "-e hello.obj -e hello.exe -e hello.obj sub1/{}".format(database_name), stdout = r"""hello.obj: %(sig_re)s \d+ \d+ %(sub1_hello_c)s: %(sig_re)s \d+ \d+ fake_cc\.py: %(sig_re)s \d+ \d+ @@ -263,7 +265,7 @@ hello.obj: %(sig_re)s \d+ \d+ %(sig_re)s \[.*\] """ % locals()) -test.run_sconsign(arguments = "sub2/.sconsign", +test.run_sconsign(arguments = "sub2/{}".format(database_name), stdout = r"""hello.c: %(sig_re)s \d+ \d+ hello.exe: %(sig_re)s \d+ \d+ %(sub2_hello_obj)s: %(sig_re)s \d+ \d+ @@ -279,7 +281,7 @@ inc1.h: %(sig_re)s \d+ \d+ inc2.h: %(sig_re)s \d+ \d+ """ % locals()) -#test.run_sconsign(arguments = "-i -v sub2/.sconsign", +#test.run_sconsign(arguments = "-i -v sub2/{}".format(database_name), # stdout = r"""hello.c: %(sig_re)s \d+ \d+ #hello.exe: %(sig_re)s \d+ \d+ # implicit: @@ -291,7 +293,7 @@ inc2.h: %(sig_re)s \d+ \d+ # inc2.h: %(sig_re)s \d+ \d+ #""" % locals()) -test.run_sconsign(arguments = "-e hello.obj sub2/.sconsign sub1/.sconsign", +test.run_sconsign(arguments = "-e hello.obj sub2/{} sub1/{}".format(database_name, database_name), stdout = r"""hello.obj: %(sig_re)s \d+ \d+ %(sub2_hello_c)s: %(sig_re)s \d+ \d+ %(sub2_inc1_h)s: %(sig_re)s \d+ \d+ diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 50280be..29bd123 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -45,6 +45,7 @@ from collections import namedtuple from TestCommon import * from TestCommon import __all__ +from SCons.Util import get_hash_format, get_current_hash_algorithm_used from TestCmd import Popen from TestCmd import PIPE @@ -719,6 +720,27 @@ class TestSCons(TestCommon): for p in patterns: result.extend(sorted(glob.glob(p))) return result + + def get_sconsignname(self): + """Get the scons database name used, and return both the prefix and full filename. + if the user left the options defaulted AND the default algorithm set by + SCons is md5, then set the database name to be the special default name + + otherwise, if it defaults to something like 'sha1' or the user explicitly + set 'md5' as the hash format, set the database name to .sconsign_<algorithm> + eg .sconsign_sha1, etc. + + Returns: + a pair containing: the current dbname, the dbname.dblite filename + """ + hash_format = get_hash_format() + current_hash_algorithm = get_current_hash_algorithm_used() + if hash_format is None and current_hash_algorithm == 'md5': + return ".sconsign" + else: + database_prefix=".sconsign_%s" % current_hash_algorithm + return database_prefix + def unlink_sconsignfile(self, name='.sconsign.dblite'): """Delete the sconsign file. @@ -1415,10 +1437,10 @@ SConscript(sconscript) for ext, flag in bld_desc: # each file in TryBuild if ext in ['.c', '.cpp']: conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\ - r'_[a-z0-9]{32}_\d+%s' % re.escape(ext) + r'_[a-z0-9]{32,64}_\d+%s' % re.escape(ext) elif ext == '': conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\ - r'_[a-z0-9]{32}(_\d+_[a-z0-9]{32})?' + r'_[a-z0-9]{32,64}(_\d+_[a-z0-9]{32,64})?' else: # We allow the second hash group to be optional because @@ -1430,7 +1452,7 @@ SConscript(sconscript) # TODO: perhaps revisit and/or fix file naming for intermediate files in # Configure context logic conf_filename = re.escape(os.path.join(sconf_dir, "conftest")) +\ - r'_[a-z0-9]{32}_\d+(_[a-z0-9]{32})?%s' % re.escape(ext) + r'_[a-z0-9]{32,64}_\d+(_[a-z0-9]{32,64})?%s' % re.escape(ext) if flag == self.NCR: # NCR = Non Cached Rebuild |