From 8934947d8d4033f31231c123feb6cdaba0db484c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 31 Aug 2023 10:50:53 -0600 Subject: Tweak Util module Move UtilTests.py from the top level to the Util package directory, for consistency with the other packages with unittests. Renamed Util/types.py -> Util/sctypes.py. types is the name of a stdlib module and it's a bad idea to duplicate it, even though in this case it's "legal" since the file was not at the top level. (Moving UtilTests.py actually made this a real problem) Class Selector is no longer an OrderedDict, it just inherits from dict as ordering is now preserved and we never used any extra features of OrderedDict. Fix API doc build - was missing a good bit of Util since it was split into a package. Moved the import-loop warning to the top of __init__.py so it will be more visible. Fiddly linting, doc-stringing, etc. Super-fiddly: pylint flags foo, bar and baz as prohibited variable/function/method names. Actually changed these in UtilTests.py. SCons.Errors has been a source of import loops because it imports Util. Now Util is split, directly import from the Util.sctypes submodule the two things Errors needs - this may reduce the chance of import problems. Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + SCons/Errors.py | 7 +- SCons/Node/FS.py | 2 +- SCons/Util/UtilTests.py | 1216 +++++++++++++++++++++++++++++++++++++++++++++ SCons/Util/__init__.py | 247 +++++---- SCons/Util/envs.py | 43 +- SCons/Util/filelock.py | 2 +- SCons/Util/hashes.py | 33 +- SCons/Util/sctypes.py | 312 ++++++++++++ SCons/Util/stats.py | 43 +- SCons/Util/types.py | 315 ------------ SCons/UtilTests.py | 1215 -------------------------------------------- doc/sphinx/SCons.Util.rst | 36 ++ doc/sphinx/SCons.rst | 9 +- doc/sphinx/index.rst | 3 +- 15 files changed, 1800 insertions(+), 1684 deletions(-) create mode 100644 SCons/Util/UtilTests.py create mode 100644 SCons/Util/sctypes.py delete mode 100644 SCons/Util/types.py delete mode 100644 SCons/UtilTests.py create mode 100644 doc/sphinx/SCons.Util.rst diff --git a/CHANGES.txt b/CHANGES.txt index 287fd18..0abd561 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -168,6 +168,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER builds. Also add a simple filesystem-based locking protocol to try to avoid the problem occuring. - Update the first two chapters on building with SCons in the User Guide. + - Some cleanup to the Util package. From Jonathon Reinhart: - Fix another instance of `int main()` in CheckLib() causing failures diff --git a/SCons/Errors.py b/SCons/Errors.py index b40ba0e..012d1c6 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -27,7 +27,8 @@ Used to handle internal and user errors in SCons. """ import shutil -import SCons.Util + +from SCons.Util.sctypes import to_String, is_String # Note that not all Errors are defined here, some are at the point of use @@ -77,7 +78,7 @@ class BuildError(Exception): # py3: errstr should be string and not bytes. - self.errstr = SCons.Util.to_String(errstr) + self.errstr = to_String(errstr) self.status = status self.exitstatus = exitstatus self.filename = filename @@ -189,7 +190,7 @@ def convert_to_BuildError(status, exc_info=None): status=2, exitstatus=2, exc_info=exc_info) - elif SCons.Util.is_String(status): + elif is_String(status): buildError = BuildError( errstr=status, status=2, diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 3162b21..a5282e6 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -1204,7 +1204,7 @@ class LocalFS: if hasattr(os, 'readlink'): - def readlink(self, file): + def readlink(self, file) -> str: return os.readlink(file) else: diff --git a/SCons/Util/UtilTests.py b/SCons/Util/UtilTests.py new file mode 100644 index 0000000..ff32bab --- /dev/null +++ b/SCons/Util/UtilTests.py @@ -0,0 +1,1216 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import functools +import hashlib +import io +import os +import sys +import unittest +import unittest.mock +import warnings +from collections import UserDict, UserList, UserString, namedtuple +from typing import Union + +import TestCmd + +import SCons.Errors +import SCons.compat +from SCons.Util import ( + ALLOWED_HASH_FORMATS, + AddPathIfNotExists, + AppendPath, + CLVar, + LogicalLines, + NodeList, + PrependPath, + Proxy, + Selector, + WhereIs, + adjustixes, + containsAll, + containsAny, + containsOnly, + dictify, + display, + flatten, + get_env_bool, + get_environment_var, + get_native_path, + get_os_env_bool, + hash_collect, + hash_signature, + is_Dict, + is_List, + is_String, + is_Tuple, + print_tree, + render_tree, + set_hash_format, + silent_intern, + splitext, + to_String, + to_bytes, + to_str, +) +from SCons.Util.hashes import ( + _attempt_init_of_python_3_9_hash_object, + _attempt_get_hash_function, + _get_hash_object, + _set_allowed_viable_default_hashes, +) + +# These Util classes have no unit tests. Some don't make sense to test? +# DisplayEngine, Delegate, MethodWrapper, UniqueList, Unbuffered, Null, NullSeq + + +class OutBuffer: + def __init__(self) -> None: + self.buffer = "" + + def write(self, text: str) -> None: + self.buffer = self.buffer + text + + +class dictifyTestCase(unittest.TestCase): + def test_dictify(self) -> None: + """Test the dictify() function""" + r = dictify(['a', 'b', 'c'], [1, 2, 3]) + assert r == {'a': 1, 'b': 2, 'c': 3}, r + + r = {} + dictify(['a'], [1], r) + dictify(['b'], [2], r) + dictify(['c'], [3], r) + assert r == {'a': 1, 'b': 2, 'c': 3}, r + + +class UtilTestCase(unittest.TestCase): + def test_splitext(self) -> None: + assert splitext('foo') == ('foo', '') + assert splitext('foo.bar') == ('foo', '.bar') + assert splitext(os.path.join('foo.bar', 'blat')) == (os.path.join('foo.bar', 'blat'), '') + + class Node: + def __init__(self, name, children=[]) -> None: + self.children = children + self.name = name + self.nocache = None + + def __str__(self) -> str: + return self.name + + def exists(self) -> bool: + return True + + def rexists(self) -> bool: + return True + + def has_builder(self) -> bool: + return True + + def has_explicit_builder(self) -> bool: + return True + + def side_effect(self) -> bool: + return True + + def precious(self) -> bool: + return True + + def always_build(self) -> bool: + return True + + def is_up_to_date(self) -> bool: + return True + + def noclean(self) -> bool: + return True + + def tree_case_1(self): + """Fixture for the render_tree() and print_tree() tests.""" + windows_h = self.Node("windows.h") + stdlib_h = self.Node("stdlib.h") + stdio_h = self.Node("stdio.h") + bar_c = self.Node("bar.c", [stdlib_h, windows_h]) + bar_o = self.Node("bar.o", [bar_c]) + foo_c = self.Node("foo.c", [stdio_h]) + foo_o = self.Node("foo.o", [foo_c]) + prog = self.Node("prog", [foo_o, bar_o]) + + expect = """\ ++-prog + +-foo.o + | +-foo.c + | +-stdio.h + +-bar.o + +-bar.c + +-stdlib.h + +-windows.h +""" + + lines = expect.split('\n')[:-1] + lines = ['[E BSPACN ]' + l for l in lines] + withtags = '\n'.join(lines) + '\n' + + return prog, expect, withtags + + def tree_case_2(self, prune: int=1): + """Fixture for the render_tree() and print_tree() tests.""" + + types_h = self.Node('types.h') + malloc_h = self.Node('malloc.h') + stdlib_h = self.Node('stdlib.h', [types_h, malloc_h]) + bar_h = self.Node('bar.h', [stdlib_h]) + blat_h = self.Node('blat.h', [stdlib_h]) + blat_c = self.Node('blat.c', [blat_h, bar_h]) + blat_o = self.Node('blat.o', [blat_c]) + + expect = """\ ++-blat.o + +-blat.c + +-blat.h + | +-stdlib.h + | +-types.h + | +-malloc.h + +-bar.h +""" + if prune: + expect += """ +-[stdlib.h] +""" + else: + expect += """ +-stdlib.h + +-types.h + +-malloc.h +""" + + lines = expect.split('\n')[:-1] + lines = ['[E BSPACN ]' + l for l in lines] + withtags = '\n'.join(lines) + '\n' + + return blat_o, expect, withtags + + def test_render_tree(self) -> None: + """Test the render_tree() function""" + + def get_children(node): + return node.children + + node, expect, withtags = self.tree_case_1() + actual = render_tree(node, get_children) + assert expect == actual, (expect, actual) + + node, expect, withtags = self.tree_case_2() + actual = render_tree(node, get_children, 1) + assert expect == actual, (expect, actual) + + # Ensure that we can call render_tree on the same Node + # again. This wasn't possible in version 2.4.1 and earlier + # due to a bug in render_tree (visited was set to {} as default + # parameter) + actual = render_tree(node, get_children, 1) + assert expect == actual, (expect, actual) + + def test_print_tree(self) -> None: + """Test the print_tree() function""" + + def get_children(node): + return node.children + + save_stdout = sys.stdout + + try: + node, expect, withtags = self.tree_case_1() + + IOStream = io.StringIO + sys.stdout = IOStream() + print_tree(node, get_children) + actual = sys.stdout.getvalue() + assert expect == actual, (expect, actual) + + sys.stdout = IOStream() + print_tree(node, get_children, showtags=1) + actual = sys.stdout.getvalue() + assert withtags == actual, (withtags, actual) + + # Test that explicitly setting prune to zero works + # the same as the default (see above) + node, expect, withtags = self.tree_case_2(prune=0) + + sys.stdout = IOStream() + print_tree(node, get_children, 0) + actual = sys.stdout.getvalue() + assert expect == actual, (expect, actual) + + sys.stdout = IOStream() + print_tree(node, get_children, 0, showtags=1) + actual = sys.stdout.getvalue() + assert withtags == actual, (withtags, actual) + + # Test output with prune=1 + node, expect, withtags = self.tree_case_2(prune=1) + + sys.stdout = IOStream() + print_tree(node, get_children, 1) + actual = sys.stdout.getvalue() + assert expect == actual, (expect, actual) + + # Ensure that we can call print_tree on the same Node + # again. This wasn't possible in version 2.4.1 and earlier + # due to a bug in print_tree (visited was set to {} as default + # parameter) + sys.stdout = IOStream() + print_tree(node, get_children, 1) + actual = sys.stdout.getvalue() + assert expect == actual, (expect, actual) + + sys.stdout = IOStream() + print_tree(node, get_children, 1, showtags=1) + actual = sys.stdout.getvalue() + assert withtags == actual, (withtags, actual) + finally: + sys.stdout = save_stdout + + def test_is_Dict(self) -> None: + assert is_Dict({}) + assert is_Dict(UserDict()) + try: + class mydict(dict): + pass + except TypeError: + pass + else: + assert is_Dict(mydict({})) + assert not is_Dict([]) + assert not is_Dict(()) + assert not is_Dict("") + + + def test_is_List(self) -> None: + assert is_List([]) + assert is_List(UserList()) + try: + class mylist(list): + pass + except TypeError: + pass + else: + assert is_List(mylist([])) + assert not is_List(()) + assert not is_List({}) + assert not is_List("") + + def test_is_String(self) -> None: + assert is_String("") + assert is_String(UserString('')) + try: + class mystr(str): + pass + except TypeError: + pass + else: + assert is_String(mystr('')) + assert not is_String({}) + assert not is_String([]) + assert not is_String(()) + + def test_is_Tuple(self) -> None: + assert is_Tuple(()) + try: + class mytuple(tuple): + pass + except TypeError: + pass + else: + assert is_Tuple(mytuple(())) + assert not is_Tuple([]) + assert not is_Tuple({}) + assert not is_Tuple("") + + def test_to_Bytes(self) -> None: + """ Test the to_Bytes method""" + self.assertEqual(to_bytes('Hello'), + bytearray('Hello', 'utf-8'), + "Check that to_bytes creates byte array when presented with non byte string.") + + def test_to_String(self) -> None: + """Test the to_String() method.""" + assert to_String(1) == "1", to_String(1) + assert to_String([1, 2, 3]) == str([1, 2, 3]), to_String([1, 2, 3]) + assert to_String("foo") == "foo", to_String("foo") + assert to_String(None) == 'None' + # test low level string converters too + assert to_str(None) == 'None' + assert to_bytes(None) == b'None' + + s1 = UserString('blah') + assert to_String(s1) == s1, s1 + assert to_String(s1) == 'blah', s1 + + class Derived(UserString): + pass + + s2 = Derived('foo') + assert to_String(s2) == s2, s2 + assert to_String(s2) == 'foo', s2 + + + def test_WhereIs(self) -> None: + test = TestCmd.TestCmd(workdir='') + + sub1_xxx_exe = test.workpath('sub1', 'xxx.exe') + sub2_xxx_exe = test.workpath('sub2', 'xxx.exe') + sub3_xxx_exe = test.workpath('sub3', 'xxx.exe') + sub4_xxx_exe = test.workpath('sub4', 'xxx.exe') + + test.subdir('subdir', 'sub1', 'sub2', 'sub3', 'sub4') + + if sys.platform != 'win32': + test.write(sub1_xxx_exe, "\n") + + os.mkdir(sub2_xxx_exe) + + test.write(sub3_xxx_exe, "\n") + os.chmod(sub3_xxx_exe, 0o777) + + test.write(sub4_xxx_exe, "\n") + os.chmod(sub4_xxx_exe, 0o777) + + env_path = os.environ['PATH'] + + try: + pathdirs_1234 = [test.workpath('sub1'), + test.workpath('sub2'), + test.workpath('sub3'), + test.workpath('sub4'), + ] + env_path.split(os.pathsep) + + pathdirs_1243 = [test.workpath('sub1'), + test.workpath('sub2'), + test.workpath('sub4'), + test.workpath('sub3'), + ] + env_path.split(os.pathsep) + + os.environ['PATH'] = os.pathsep.join(pathdirs_1234) + wi = WhereIs('xxx.exe') + assert wi == test.workpath(sub3_xxx_exe), wi + wi = WhereIs('xxx.exe', pathdirs_1243) + assert wi == test.workpath(sub4_xxx_exe), wi + wi = WhereIs('xxx.exe', os.pathsep.join(pathdirs_1243)) + assert wi == test.workpath(sub4_xxx_exe), wi + + wi = WhereIs('xxx.exe', reject=sub3_xxx_exe) + assert wi == test.workpath(sub4_xxx_exe), wi + wi = WhereIs('xxx.exe', pathdirs_1243, reject=sub3_xxx_exe) + assert wi == test.workpath(sub4_xxx_exe), wi + + os.environ['PATH'] = os.pathsep.join(pathdirs_1243) + wi = WhereIs('xxx.exe') + assert wi == test.workpath(sub4_xxx_exe), wi + wi = WhereIs('xxx.exe', pathdirs_1234) + assert wi == test.workpath(sub3_xxx_exe), wi + wi = WhereIs('xxx.exe', os.pathsep.join(pathdirs_1234)) + assert wi == test.workpath(sub3_xxx_exe), wi + + if sys.platform == 'win32': + wi = WhereIs('xxx', pathext='') + assert wi is None, wi + + wi = WhereIs('xxx', pathext='.exe') + assert wi == test.workpath(sub4_xxx_exe), wi + + wi = WhereIs('xxx', path=pathdirs_1234, pathext='.BAT;.EXE') + assert wi.lower() == test.workpath(sub3_xxx_exe).lower(), wi + + # Test that we return a normalized path even when + # the path contains forward slashes. + forward_slash = test.workpath('') + '/sub3' + wi = WhereIs('xxx', path=forward_slash, pathext='.EXE') + assert wi.lower() == test.workpath(sub3_xxx_exe).lower(), wi + + del os.environ['PATH'] + wi = WhereIs('xxx.exe') + assert wi is None, wi + + finally: + os.environ['PATH'] = env_path + + def test_get_env_var(self) -> None: + """Testing get_environment_var().""" + assert get_environment_var("$FOO") == "FOO", get_environment_var("$FOO") + assert get_environment_var("${BAR}") == "BAR", get_environment_var("${BAR}") + assert get_environment_var("$FOO_BAR1234") == "FOO_BAR1234", get_environment_var("$FOO_BAR1234") + assert get_environment_var("${BAR_FOO1234}") == "BAR_FOO1234", get_environment_var("${BAR_FOO1234}") + assert get_environment_var("${BAR}FOO") is None, get_environment_var("${BAR}FOO") + assert get_environment_var("$BAR ") is None, get_environment_var("$BAR ") + assert get_environment_var("FOO$BAR") is None, get_environment_var("FOO$BAR") + assert get_environment_var("$FOO[0]") is None, get_environment_var("$FOO[0]") + assert get_environment_var("${some('complex expression')}") is None, get_environment_var( + "${some('complex expression')}") + + def test_Proxy(self) -> None: + """Test generic Proxy class.""" + + class Subject: + def meth(self) -> int: + return 1 + + def other(self) -> int: + return 2 + + s = Subject() + s.attr = 3 + + class ProxyTest(Proxy): + def other(self) -> int: + return 4 + + p = ProxyTest(s) + + assert p.meth() == 1, p.meth() + assert p.other() == 4, p.other() + assert p.attr == 3, p.attr + + p.attr = 5 + s.attr = 6 + + assert p.attr == 5, p.attr + assert p.get() == s, p.get() + + def test_display(self) -> None: + old_stdout = sys.stdout + sys.stdout = OutBuffer() + display("line1") + display.set_mode(0) + display("line2") + display.set_mode(1) + display("line3") + display("line4\n", append_newline=0) + display.set_mode(0) + display("dont print1") + display("dont print2\n", append_newline=0) + display.set_mode(1) + assert sys.stdout.buffer == "line1\nline3\nline4\n" + sys.stdout = old_stdout + + def test_get_native_path(self) -> None: + """Test the get_native_path() function.""" + import tempfile + f, filename = tempfile.mkstemp(text=True) + os.close(f) + data = '1234567890 ' + filename + try: + with open(filename, 'w') as file: + file.write(data) + with open(get_native_path(filename), 'r') as native: + assert native.read() == data + finally: + try: + os.unlink(filename) + except OSError: + pass + + def test_PrependPath(self) -> None: + """Test prepending to a path""" + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + p2: Union[list, str] = r'C:\mydir\num\one;C:\mydir\num\two' + # have to include the pathsep here so that the test will work on UNIX too. + p1 = PrependPath(p1, r'C:\dir\num\two', sep=';') + p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') + assert p1 == r'C:\dir\num\three;C:\dir\num\two;C:\dir\num\one', p1 + + p2 = PrependPath(p2, r'C:\mydir\num\three', sep=';') + p2 = PrependPath(p2, r'C:\mydir\num\one', sep=';') + assert p2 == r'C:\mydir\num\one;C:\mydir\num\three;C:\mydir\num\two', p2 + + # check (only) first one is kept if there are dupes in new + p3: Union[list, str] = r'C:\dir\num\one' + p3 = PrependPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') + assert p3 == r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one', p3 + + def test_AppendPath(self) -> None: + """Test appending to a path.""" + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + p2: Union[list, str] = r'C:\mydir\num\one;C:\mydir\num\two' + # have to include the pathsep here so that the test will work on UNIX too. + p1 = AppendPath(p1, r'C:\dir\num\two', sep=';') + p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') + assert p1 == r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three', p1 + + p2 = AppendPath(p2, r'C:\mydir\num\three', sep=';') + p2 = AppendPath(p2, r'C:\mydir\num\one', sep=';') + assert p2 == r'C:\mydir\num\two;C:\mydir\num\three;C:\mydir\num\one', p2 + + # check (only) last one is kept if there are dupes in new + p3: Union[list, str] = r'C:\dir\num\one' + p3 = AppendPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') + assert p3 == r'C:\dir\num\one;C:\dir\num\three;C:\dir\num\two', p3 + + def test_PrependPathPreserveOld(self) -> None: + """Test prepending to a path while preserving old paths""" + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + # have to include the pathsep here so that the test will work on UNIX too. + p1 = PrependPath(p1, r'C:\dir\num\two', sep=';', delete_existing=False) + p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') + assert p1 == r'C:\dir\num\three;C:\dir\num\one;C:\dir\num\two', p1 + + def test_AppendPathPreserveOld(self) -> None: + """Test appending to a path while preserving old paths""" + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + # have to include the pathsep here so that the test will work on UNIX too. + p1 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=False) + p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') + assert p1 == r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three', p1 + + def test_addPathIfNotExists(self) -> None: + """Test the AddPathIfNotExists() function""" + env_dict = {'FOO': os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), + 'BAR': os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), + 'BLAT': [os.path.normpath('/foo/bar'), + os.path.normpath('/baz/blat')]} + AddPathIfNotExists(env_dict, 'FOO', os.path.normpath('/foo/bar')) + AddPathIfNotExists(env_dict, 'BAR', os.path.normpath('/bar/foo')) + AddPathIfNotExists(env_dict, 'BAZ', os.path.normpath('/foo/baz')) + AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/blat')) + AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/foo')) + + assert env_dict['FOO'] == os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), env_dict['FOO'] + assert env_dict['BAR'] == os.path.normpath('/bar/foo') + os.pathsep + \ + os.path.normpath('/foo/bar') + os.pathsep + \ + os.path.normpath('/baz/blat'), env_dict['BAR'] + assert env_dict['BAZ'] == os.path.normpath('/foo/baz'), env_dict['BAZ'] + assert env_dict['BLAT'] == [os.path.normpath('/baz/foo'), + os.path.normpath('/foo/bar'), + os.path.normpath('/baz/blat')], env_dict['BLAT'] + + def test_CLVar(self) -> None: + """Test the command-line construction variable class""" + + # the default value should be an empty list + d = CLVar() + assert isinstance(d, CLVar), type(d) + assert d.data == [], d.data + assert str(d) == '', str(d) + + # input to CLVar is a string - should be split + f = CLVar('aa bb') + + r = f + 'cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ' cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', ' cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', ' cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + # input to CLVar is a list of one string, should not be split + f = CLVar(['aa bb']) + + r = f + 'cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ' cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', 'cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', ' cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa bb', ' cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + # input to CLVar is a list of strings + f = CLVar(['aa', 'bb']) + + r = f + 'cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ' cc dd' + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', ' cc dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + ['cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + r = f + [' cc', 'dd'] + assert isinstance(r, CLVar), type(r) + assert r.data == ['aa', 'bb', ' cc', 'dd'], r.data + assert str(r) == 'aa bb cc dd', str(r) + + # make sure inplace adding a string works as well (issue 2399) + # UserList would convert the string to a list of chars + f = CLVar(['aa', 'bb']) + f += 'cc dd' + assert isinstance(f, CLVar), type(f) + assert f.data == ['aa', 'bb', 'cc', 'dd'], f.data + assert str(f) == 'aa bb cc dd', str(f) + + f = CLVar(['aa', 'bb']) + f += ' cc dd' + assert isinstance(f, CLVar), type(f) + assert f.data == ['aa', 'bb', 'cc', 'dd'], f.data + assert str(f) == 'aa bb cc dd', str(f) + + + def test_Selector(self) -> None: + """Test the Selector class""" + + class MyNode: + def __init__(self, name) -> None: + self.name = name + + def __str__(self) -> str: + return self.name + + def get_suffix(self): + return os.path.splitext(self.name)[1] + + s = Selector({'a': 'AAA', 'b': 'BBB'}) + assert s['a'] == 'AAA', s['a'] + assert s['b'] == 'BBB', s['b'] + exc_caught = None + try: + x = s['c'] + except KeyError: + exc_caught = 1 + assert exc_caught, "should have caught a KeyError" + s['c'] = 'CCC' + assert s['c'] == 'CCC', s['c'] + + class DummyEnv(UserDict): + def subst(self, key): + if key[0] == '$': + return self[key[1:]] + return key + + env = DummyEnv() + + s = Selector({'.d': 'DDD', '.e': 'EEE'}) + ret = s(env, []) + assert ret is None, ret + ret = s(env, [MyNode('foo.d')]) + assert ret == 'DDD', ret + ret = s(env, [MyNode('bar.e')]) + assert ret == 'EEE', ret + ret = s(env, [MyNode('bar.x')]) + assert ret is None, ret + s[None] = 'XXX' + ret = s(env, [MyNode('bar.x')]) + assert ret == 'XXX', ret + + env = DummyEnv({'FSUFF': '.f', 'GSUFF': '.g'}) + + s = Selector({'$FSUFF': 'FFF', '$GSUFF': 'GGG'}) + ret = s(env, [MyNode('foo.f')]) + assert ret == 'FFF', ret + ret = s(env, [MyNode('bar.g')]) + assert ret == 'GGG', ret + + def test_adjustixes(self) -> None: + """Test the adjustixes() function""" + r = adjustixes('file', 'pre-', '-suf') + assert r == 'pre-file-suf', r + r = adjustixes('pre-file', 'pre-', '-suf') + assert r == 'pre-file-suf', r + r = adjustixes('file-suf', 'pre-', '-suf') + assert r == 'pre-file-suf', r + r = adjustixes('pre-file-suf', 'pre-', '-suf') + assert r == 'pre-file-suf', r + r = adjustixes('pre-file.xxx', 'pre-', '-suf') + assert r == 'pre-file.xxx', r + r = adjustixes('dir/file', 'pre-', '-suf') + assert r == os.path.join('dir', 'pre-file-suf'), r + + # Verify that the odd case when library name is specified as 'lib' + # doesn't yield lib.so, but yields the expected liblib.so + r = adjustixes('PREFIX', 'PREFIX', 'SUFFIX') + assert r == 'PREFIXPREFIXSUFFIX', "Failed handling when filename = PREFIX [r='%s']" % r + + def test_containsAny(self) -> None: + """Test the containsAny() function""" + assert containsAny('*.py', '*?[]') + assert not containsAny('file.txt', '*?[]') + + def test_containsAll(self) -> None: + """Test the containsAll() function""" + assert containsAll('43221', '123') + assert not containsAll('134', '123') + + def test_containsOnly(self) -> None: + """Test the containsOnly() function""" + assert containsOnly('.83', '0123456789.') + assert not containsOnly('43221', '123') + + def test_LogicalLines(self) -> None: + """Test the LogicalLines class""" + content = """ +foo \\ +bar \\ +baz +foo +bling \\ +bling \\ bling +bling +""" + fobj = io.StringIO(content) + lines = LogicalLines(fobj).readlines() + assert lines == [ + '\n', + 'foo bar baz\n', + 'foo\n', + 'bling bling \\ bling\n', + 'bling\n', + ], lines + + def test_intern(self) -> None: + s1 = silent_intern("spam") + s3 = silent_intern(42) + s4 = silent_intern("spam") + assert id(s1) == id(s4) + + +class HashTestCase(unittest.TestCase): + + def test_collect(self) -> None: + """Test collecting a list of signatures into a new signature value + """ + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + '8980c988edc2c78cc43ccb718c06efd5', + '53fd88c84ff8a285eb6e0a687e55b8c7'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '42eda1b5dcb3586bccfb1c69f22f923145271d97', + '2eb2f7be4e883ebe52034281d818c91e1cf16256'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', + 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), + }.items(): + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + hs = functools.partial(hash_signature, hash_format=algorithm) + s = list(map(hs, ('111', '222', '333'))) + + assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) + assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) + assert expected[2] == hash_collect(s, hash_format=algorithm) + + def test_MD5signature(self) -> None: + """Test generating a signature""" + for algorithm, expected in { + 'md5': ('698d51a19d8a121ce581499d7b701668', + 'bcbe3365e6ac95ea2c0343a2395834dd'), + 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', + '1c6637a8f2e1f75e06ff9984894d6bd16a3a36a9'), + 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', + '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), + }.items(): + # if the current platform does not support the algorithm we're looking at, + # skip the test steps for that algorithm, but display a warning to the user + if algorithm not in ALLOWED_HASH_FORMATS: + warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) + else: + s = hash_signature('111', hash_format=algorithm) + assert expected[0] == s, s + + s = hash_signature('222', hash_format=algorithm) + assert expected[1] == s, s + +# this uses mocking out, which is platform specific, however, the FIPS +# behavior this is testing is also platform-specific, and only would be +# visible in hosts running Linux with the fips_mode kernel flag along +# with using OpenSSL. + +class FIPSHashTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + ############################### + # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 + self.fake_md5=lambda usedforsecurity=True: (usedforsecurity, 'md5') + self.fake_sha1=lambda usedforsecurity=True: (usedforsecurity, 'sha1') + self.fake_sha256=lambda usedforsecurity=True: (usedforsecurity, 'sha256') + ############################### + + ############################### + # hashlib mocks + md5Available = unittest.mock.Mock(md5=self.fake_md5) + del md5Available.sha1 + del md5Available.sha256 + self.md5Available=md5Available + + md5Default = unittest.mock.Mock(md5=self.fake_md5, sha1=self.fake_sha1) + del md5Default.sha256 + self.md5Default=md5Default + + sha1Default = unittest.mock.Mock(sha1=self.fake_sha1, sha256=self.fake_sha256) + del sha1Default.md5 + self.sha1Default=sha1Default + + sha256Default = unittest.mock.Mock(sha256=self.fake_sha256, **{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError}) + self.sha256Default=sha256Default + + all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) + self.all_throw=all_throw + + no_algorithms = unittest.mock.Mock() + del no_algorithms.md5 + del no_algorithms.sha1 + del no_algorithms.sha256 + del no_algorithms.nonexist + self.no_algorithms=no_algorithms + + unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) + del unsupported_algorithm.md5 + del unsupported_algorithm.sha1 + del unsupported_algorithm.sha256 + del unsupported_algorithm.unsupported + self.unsupported_algorithm=unsupported_algorithm + ############################### + + ############################### + # system version mocks + VersionInfo = namedtuple('VersionInfo', 'major minor micro releaselevel serial') + v3_8 = VersionInfo(3, 8, 199, 'super-beta', 1337) + v3_9 = VersionInfo(3, 9, 0, 'alpha', 0) + v4_8 = VersionInfo(4, 8, 0, 'final', 0) + + self.sys_v3_8 = unittest.mock.Mock(version_info=v3_8) + self.sys_v3_9 = unittest.mock.Mock(version_info=v3_9) + self.sys_v4_8 = unittest.mock.Mock(version_info=v4_8) + ############################### + + def test_basic_failover_bad_hashlib_hash_init(self) -> None: + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash init function returns None""" + assert _attempt_init_of_python_3_9_hash_object(None) is None + + def test_basic_failover_bad_hashlib_hash_get(self) -> None: + """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), + the hash get function returns None""" + assert _attempt_get_hash_function("nonexist", self.no_algorithms) is None + + def test_usedforsecurity_flag_behavior(self) -> None: + """Test usedforsecurity flag -> should be set to 'True' on older versions of python, and 'False' on Python >= 3.9""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected + + def test_automatic_default_to_md5(self) -> None: + """Test automatic default to md5 even if sha1 available""" + for version, expected in { + self.sys_v3_8: (True, 'md5'), + self.sys_v3_9: (False, 'md5'), + self.sys_v4_8: (False, 'md5'), + }.items(): + _set_allowed_viable_default_hashes(self.md5Default, version) + set_hash_format(None, self.md5Default, version) + assert _get_hash_object(None, self.md5Default, version) == expected + + def test_automatic_default_to_sha256(self) -> None: + """Test automatic default to sha256 if other algorithms available but throw""" + for version, expected in { + self.sys_v3_8: (True, 'sha256'), + self.sys_v3_9: (False, 'sha256'), + self.sys_v4_8: (False, 'sha256'), + }.items(): + _set_allowed_viable_default_hashes(self.sha256Default, version) + set_hash_format(None, self.sha256Default, version) + assert _get_hash_object(None, self.sha256Default, version) == expected + + def test_automatic_default_to_sha1(self) -> None: + """Test automatic default to sha1 if md5 is missing from hashlib entirely""" + for version, expected in { + self.sys_v3_8: (True, 'sha1'), + self.sys_v3_9: (False, 'sha1'), + self.sys_v4_8: (False, 'sha1'), + }.items(): + _set_allowed_viable_default_hashes(self.sha1Default, version) + set_hash_format(None, self.sha1Default, version) + assert _get_hash_object(None, self.sha1Default, version) == expected + + def test_no_available_algorithms(self) -> None: + """expect exceptions on no available algorithms or when all algorithms throw""" + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.all_throw) + self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.unsupported_algorithm) + + def test_bad_algorithm_set_attempt(self) -> None: + """expect exceptions on user setting an unsupported algorithm selections, either by host or by SCons""" + + # nonexistant hash algorithm, not supported by SCons + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) + + # md5 is default-allowed, but in this case throws when we attempt to use it + _set_allowed_viable_default_hashes(self.md5Available) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) + + # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons + _set_allowed_viable_default_hashes(self.sha1Default) + self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) + + def tearDown(self) -> None: + """Return SCons back to the normal global state for the hashing functions.""" + _set_allowed_viable_default_hashes(hashlib, sys) + set_hash_format(None) + + +class NodeListTestCase(unittest.TestCase): + def test_simple_attributes(self) -> None: + """Test simple attributes of a NodeList class""" + + class TestClass: + def __init__(self, name, child=None) -> None: + self.child = child + self.name = name + + t1 = TestClass('t1', TestClass('t1child')) + t2 = TestClass('t2', TestClass('t2child')) + t3 = TestClass('t3') + + nl = NodeList([t1, t2, t3]) + assert nl.name == ['t1', 't2', 't3'], nl.name + assert nl[0:2].child.name == ['t1child', 't2child'], \ + nl[0:2].child.name + + def test_callable_attributes(self) -> None: + """Test callable attributes of a NodeList class""" + + class TestClass: + def __init__(self, name, child=None) -> None: + self.child = child + self.name = name + + def meth(self): + return self.name + "foo" + + def getself(self): + return self + + t1 = TestClass('t1', TestClass('t1child')) + t2 = TestClass('t2', TestClass('t2child')) + t3 = TestClass('t3') + + nl = NodeList([t1, t2, t3]) + assert nl.meth() == ['t1foo', 't2foo', 't3foo'], nl.meth() + assert nl.name == ['t1', 't2', 't3'], nl.name + assert nl.getself().name == ['t1', 't2', 't3'], nl.getself().name + assert nl[0:2].child.meth() == ['t1childfoo', 't2childfoo'], \ + nl[0:2].child.meth() + assert nl[0:2].child.name == ['t1child', 't2child'], \ + nl[0:2].child.name + + def test_null(self): + """Test a null NodeList""" + nl = NodeList([]) + r = str(nl) + assert r == '', r + for node in nl: + raise Exception("should not enter this loop") + + +class flattenTestCase(unittest.TestCase): + + def test_scalar(self) -> None: + """Test flattening a scalar""" + result = flatten('xyz') + self.assertEqual(result, ['xyz'], result) + + def test_dictionary_values(self) -> None: + """Test flattening the dictionary values""" + items = {"a": 1, "b": 2, "c": 3} + result = flatten(items.values()) + self.assertEqual(sorted(result), [1, 2, 3]) + + +class OsEnviron: + """Used to temporarily mock os.environ""" + + def __init__(self, environ) -> None: + self._environ = environ + + def start(self) -> None: + self._stored = os.environ + os.environ = self._environ + + def stop(self) -> None: + os.environ = self._stored + del self._stored + + def __enter__(self): + self.start() + return os.environ + + def __exit__(self, *args) -> None: + self.stop() + + +class get_env_boolTestCase(unittest.TestCase): + def test_missing(self) -> None: + env = {} + var = get_env_bool(env, 'FOO') + assert var is False, "var should be False, not %s" % repr(var) + env = {'FOO': '1'} + var = get_env_bool(env, 'BAR') + assert var is False, "var should be False, not %s" % repr(var) + + def test_true(self) -> None: + for arg in ['TRUE', 'True', 'true', + 'YES', 'Yes', 'yes', + 'Y', 'y', + 'ON', 'On', 'on', + '1', '20', '-1']: + env = {'FOO': arg} + var = get_env_bool(env, 'FOO') + assert var is True, 'var should be True, not %s' % repr(var) + + def test_false(self) -> None: + for arg in ['FALSE', 'False', 'false', + 'NO', 'No', 'no', + 'N', 'n', + 'OFF', 'Off', 'off', + '0']: + env = {'FOO': arg} + var = get_env_bool(env, 'FOO', True) + assert var is False, 'var should be True, not %s' % repr(var) + + def test_default(self) -> None: + env = {'FOO': 'other'} + var = get_env_bool(env, 'FOO', True) + assert var is True, 'var should be True, not %s' % repr(var) + var = get_env_bool(env, 'FOO', False) + assert var is False, 'var should be False, not %s' % repr(var) + + +class get_os_env_boolTestCase(unittest.TestCase): + def test_missing(self) -> None: + with OsEnviron({}): + var = get_os_env_bool('FOO') + assert var is False, "var should be False, not %s" % repr(var) + with OsEnviron({'FOO': '1'}): + var = get_os_env_bool('BAR') + assert var is False, "var should be False, not %s" % repr(var) + + def test_true(self) -> None: + for arg in ['TRUE', 'True', 'true', + 'YES', 'Yes', 'yes', + 'Y', 'y', + 'ON', 'On', 'on', + '1', '20', '-1']: + with OsEnviron({'FOO': arg}): + var = get_os_env_bool('FOO') + assert var is True, 'var should be True, not %s' % repr(var) + + def test_false(self) -> None: + for arg in ['FALSE', 'False', 'false', + 'NO', 'No', 'no', + 'N', 'n', + 'OFF', 'Off', 'off', + '0']: + with OsEnviron({'FOO': arg}): + var = get_os_env_bool('FOO', True) + assert var is False, 'var should be True, not %s' % repr(var) + + def test_default(self) -> None: + with OsEnviron({'FOO': 'other'}): + var = get_os_env_bool('FOO', True) + assert var is True, 'var should be True, not %s' % repr(var) + var = get_os_env_bool('FOO', False) + assert var is False, 'var should be False, not %s' % repr(var) + + +if __name__ == "__main__": + unittest.main() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/__init__.py b/SCons/Util/__init__.py index 078414f..be2142f 100644 --- a/SCons/Util/__init__.py +++ b/SCons/Util/__init__.py @@ -21,24 +21,50 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""SCons utility functions +""" +SCons utility functions This package contains routines for use by other parts of SCons. +Candidates for inclusion here are routines that do not need other parts +of SCons (other than Util), and have a reasonable chance of being useful +in multiple places, rather then being topical only to one module/package. """ +# Warning: SCons.Util may not be able to import other parts of SCons +# globally without hitting import loops, as various modules import +# SCons.Util themselves. If a top-level import fails, try a local import. +# If local imports work, please annotate them for pylint (and for human +# readers) to know why, with: +# importstuff # pylint: disable=import-outside-toplevel +# +# Be aware that Black will break this if the annotated line is too long - +# which it almost certainly will be. It will split it like this: +# from SCons.Errors import ( +# SConsEnvironmentError, +# ) # pylint: disable=import-outside-toplevel +# That's syntactically valid as far as Python goes, but pylint will not +# recorgnize the annotation comment unless it's on the first line, like: +# from SCons.Errors import ( # pylint: disable=import-outside-toplevel +# SConsEnvironmentError, +# ) +# (issue filed on this upstream, for now just be aware) + import copy import hashlib +import logging import os import re import sys import time -from collections import UserDict, UserList, OrderedDict, deque +from collections import UserDict, UserList, deque from contextlib import suppress from types import MethodType, FunctionType -from typing import Optional, Union +from typing import Optional, Union, Any, List from logging import Formatter -from .types import ( +# Util split into a package. Make sure things that used to work +# when importing just Util itself still work: +from .sctypes import ( DictTypes, ListTypes, SequenceTypes, @@ -83,24 +109,6 @@ from .envs import ( ) from .filelock import FileLock, SConsLockFailure - -# Note: the Util package cannot import other parts of SCons globally without -# hitting import loops. Both of these modules import SCons.Util early on, -# and are imported in many other modules: -# --> SCons.Warnings -# --> SCons.Errors -# If you run into places that have to do local imports for this reason, -# annotate them for pylint and for human readers to know why: -# pylint: disable=import-outside-toplevel -# Be aware that Black can break this if the annotated line is too -# long and it wants to split: -# from SCons.Errors import ( -# SConsEnvironmentError, -# ) # pylint: disable=import-outside-toplevel -# That's syntactically valid, but pylint won't recorgnize it with the -# annotation at the end, it would have to be on the first line -# (issues filed upstream, for now just be aware) - PYPY = hasattr(sys, 'pypy_translation_info') # this string will be hashed if a Node refers to a file that doesn't exist @@ -238,20 +246,27 @@ class DisplayEngine: display = DisplayEngine() -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) -def render_tree(root, child_func, prune: int=0, margin=[0], visited=None) -> str: +# TODO: check if this could cause problems +# pylint: disable=dangerous-default-value +def render_tree( + root, + child_func, + prune: bool = False, + margin: List[bool] = [False], + visited: Optional[dict] = None, +) -> str: """Render a tree of nodes into an ASCII tree view. Args: root: the root node of the tree child_func: the function called to get the children of a node prune: don't visit the same node twice - margin: the format of the left margin to use for children of `root`. - 1 results in a pipe, and 0 results in no pipe. + margin: the format of the left margin to use for children of *root*. + Each entry represents a column where a true value will display + a vertical bar and a false one a blank. visited: a dictionary of visited nodes in the current branch if - `prune` is 0, or in the whole tree if `prune` is 1. + *prune* is false, or in the whole tree if *prune* is true. """ - rname = str(root) # Initialize 'visited' dict, if required @@ -275,7 +290,7 @@ def render_tree(root, child_func, prune: int=0, margin=[0], visited=None) -> str visited[rname] = True for i, child in enumerate(children): - margin.append(i < len(children)-1) + margin.append(i < len(children) - 1) retval = retval + render_tree(child, child_func, prune, margin, visited) margin.pop() @@ -299,14 +314,15 @@ BOX_VERT_RIGHT = chr(0x251c) # '├' BOX_HORIZ_DOWN = chr(0x252c) # '┬' -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) +# TODO: check if this could cause problems +# pylint: disable=dangerous-default-value def print_tree( root, child_func, - prune: int=0, - showtags: bool=False, - margin=[0], - visited=None, + prune: bool = False, + showtags: int = 0, + margin: List[bool] = [False], + visited: Optional[dict] = None, lastChild: bool = False, singleLineDraw: bool = False, ) -> None: @@ -321,10 +337,13 @@ def print_tree( child_func: the function called to get the children of a node prune: don't visit the same node twice showtags: print status information to the left of each node line + The default is false (value 0). A value of 2 will also print + a legend for the margin tags. margin: the format of the left margin to use for children of *root*. - 1 results in a pipe, and 0 results in no pipe. + Each entry represents a column, where a true value will display + a vertical bar and a false one a blank. visited: a dictionary of visited nodes in the current branch if - *prune* is 0, or in the whole tree if *prune* is 1. + *prune* is false, or in the whole tree if *prune* is true. lastChild: this is the last leaf of a branch singleLineDraw: use line-drawing characters rather than ASCII. """ @@ -405,7 +424,7 @@ def print_tree( # if this item has children: if children: - margin.append(1) # Initialize margin with 1 for vertical bar. + margin.append(True) # Initialize margin for vertical bar. idx = IDX(showtags) _child = 0 # Initialize this for the first child. for C in children[:-1]: @@ -421,19 +440,19 @@ def print_tree( singleLineDraw, ) # margins are with space (index 0) because we arrived to the last child. - margin[-1] = 0 + margin[-1] = False # for this call child and nr of children needs to be set 0, to signal the second phase. print_tree(children[-1], child_func, prune, idx, margin, visited, True, singleLineDraw) margin.pop() # destroy the last margin added -def do_flatten( +def do_flatten( # pylint: disable=redefined-outer-name,redefined-builtin sequence, result, isinstance=isinstance, StringTypes=StringTypes, SequenceTypes=SequenceTypes, -) -> None: # pylint: disable=redefined-outer-name,redefined-builtin +) -> None: for item in sequence: if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): result.append(item) @@ -538,7 +557,7 @@ def semi_deepcopy(obj): class Proxy: """A simple generic Proxy class, forwarding all calls to subject. - This means you can take an object, let's call it `'obj_a`, + This means you can take an object, let's call it `'obj_a``, and wrap it in this Proxy class, with a statement like this:: proxy_obj = Proxy(obj_a) @@ -548,14 +567,15 @@ class Proxy: x = proxy_obj.var1 since the :class:`Proxy` class does not have a :attr:`var1` attribute - (but presumably `objA` does), the request actually is equivalent to saying:: + (but presumably ``obj_a`` does), the request actually is equivalent + to saying:: x = obj_a.var1 Inherit from this class to create a Proxy. With Python 3.5+ this does *not* work transparently - for :class:`Proxy` subclasses that use special .__*__() method names, + for :class:`Proxy` subclasses that use special dunder method names, because those names are now bound to the class, not the individual instances. You now need to know in advance which special method names you want to pass on to the underlying Proxy object, and specifically delegate @@ -759,14 +779,14 @@ else: f = os.path.join(p, file) if os.path.isfile(f): try: - st = os.stat(f) + mode = os.stat(f).st_mode except OSError: # os.stat() raises OSError, not IOError if the file # doesn't exist, so in this case we let IOError get # raised so as to not mask possibly serious disk or # network issues. continue - if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: + if stat.S_IXUSR & mode: try: reject.index(f) except ValueError: @@ -775,46 +795,61 @@ else: return None WhereIs.__doc__ = """\ -Return the path to an executable that matches `file`. - -Searches the given `path` for `file`, respecting any filename -extensions `pathext` (on the Windows platform only), and -returns the full path to the matching command. If no -command is found, return ``None``. - -If `path` is not specified, :attr:`os.environ[PATH]` is used. -If `pathext` is not specified, :attr:`os.environ[PATHEXT]` -is used. Will not select any path name or names in the optional -`reject` list. +Return the path to an executable that matches *file*. + +Searches the given *path* for *file*, considering any filename +extensions in *pathext* (on the Windows platform only), and +returns the full path to the matching command of the first match, +or ``None`` if there are no matches. +Will not select any path name or names in the optional +*reject* list. + +If *path* is ``None`` (the default), :attr:`os.environ[PATH]` is used. +On Windows, If *pathext* is ``None`` (the default), +:attr:`os.environ[PATHEXT]` is used. + +The construction environment method of the same name wraps a +call to this function by filling in *path* from the execution +environment if it is ``None`` (and for *pathext* on Windows, +if necessary), so if called from there, this function +will not backfill from :attr:`os.environ`. + +Note: + Finding things in :attr:`os.environ` may answer the question + "does *file* exist on the system", but not the question + "can SCons use that executable", unless the path element that + yields the match is also in the the Execution Environment + (e.g. ``env['ENV']['PATH']``). Since this utility function has no + environment reference, it cannot make that determination. """ if sys.platform == 'cygwin': import subprocess # pylint: disable=import-outside-toplevel - def get_native_path(path) -> str: + def get_native_path(path: str) -> str: cp = subprocess.run(('cygpath', '-w', path), check=False, stdout=subprocess.PIPE) return cp.stdout.decode().replace('\n', '') else: - def get_native_path(path) -> str: + def get_native_path(path: str) -> str: return path get_native_path.__doc__ = """\ Transform an absolute path into a native path for the system. In Cygwin, this converts from a Cygwin path to a Windows path, -without regard to whether `path` refers to an existing file -system object. For other platforms, `path` is unchanged. +without regard to whether *path* refers to an existing file +system object. For other platforms, *path* is unchanged. """ def Split(arg) -> list: """Returns a list of file names or other objects. - If `arg` is a string, it will be split on strings of white-space - characters within the string. If `arg` is already a list, the list - will be returned untouched. If `arg` is any other type of object, - it will be returned as a list containing just the object. + If *arg* is a string, it will be split on whitespace + within the string. If *arg* is already a list, the list + will be returned untouched. If *arg* is any other type of object, + it will be returned in a single-item list. >>> print(Split(" this is a string ")) ['this', 'is', 'a', 'string'] @@ -854,9 +889,11 @@ class CLVar(UserList): >>> c = CLVar("--some --opts and args") >>> print(len(c), repr(c)) 4 ['--some', '--opts', 'and', 'args'] - >>> c += " strips spaces " + >>> c += " strips spaces " >>> print(len(c), repr(c)) 6 ['--some', '--opts', 'and', 'args', 'strips', 'spaces'] + >>> c += [" does not split or strip "] + 7 ['--some', '--opts', 'and', 'args', 'strips', 'spaces', ' does not split or strip '] """ def __init__(self, initlist=None) -> None: @@ -877,10 +914,13 @@ class CLVar(UserList): return ' '.join([str(d) for d in self.data]) -class Selector(OrderedDict): - """A callable ordered dictionary that maps file suffixes to - dictionary values. We preserve the order in which items are added - so that :func:`get_suffix` calls always return the first suffix added. +class Selector(dict): + """A callable dict for file suffix lookup. + + Often used to associate actions or emitters with file types. + + Depends on insertion order being preserved so that :meth:`get_suffix` + calls always return the first suffix added. """ def __call__(self, env, source, ext=None): if ext is None: @@ -890,7 +930,7 @@ class Selector(OrderedDict): ext = "" try: return self[ext] - except KeyError: + except KeyError as exc: # Try to perform Environment substitution on the keys of # the dictionary before giving up. s_dict = {} @@ -902,7 +942,7 @@ class Selector(OrderedDict): # to the same suffix. If one suffix is literal # and a variable suffix contains this literal, # the literal wins and we don't raise an error. - raise KeyError(s_dict[s_k][0], k, s_k) + raise KeyError(s_dict[s_k][0], k, s_k) from exc s_dict[s_k] = (k, v) try: return s_dict[ext][1] @@ -916,13 +956,16 @@ class Selector(OrderedDict): if sys.platform == 'cygwin': # On Cygwin, os.path.normcase() lies, so just report back the # fact that the underlying Windows OS is case-insensitive. - def case_sensitive_suffixes(s1, s2) -> bool: # pylint: disable=unused-argument + def case_sensitive_suffixes(s1: str, s2: str) -> bool: # pylint: disable=unused-argument return False else: - def case_sensitive_suffixes(s1, s2) -> bool: + def case_sensitive_suffixes(s1: str, s2: str) -> bool: return os.path.normcase(s1) != os.path.normcase(s2) +case_sensitive_suffixes.__doc__ = """\ +Returns whether platform distinguishes case in file suffixes.""" + def adjustixes(fname, pre, suf, ensure_suffix: bool=False) -> str: """Adjust filename prefixes and suffixes as needed. @@ -958,6 +1001,17 @@ def adjustixes(fname, pre, suf, ensure_suffix: bool=False) -> str: def unique(seq): """Return a list of the elements in seq without duplicates, ignoring order. + For best speed, all sequence elements should be hashable. Then + :func:`unique` will usually work in linear time. + + If not possible, the sequence elements should enjoy a total + ordering, and if ``list(s).sort()`` doesn't raise ``TypeError`` + it is assumed that they do enjoy a total ordering. Then + :func:`unique` will usually work in O(N*log2(N)) time. + + If that's not possible either, the sequence elements must support + equality-testing. Then :func:`unique` will usually work in quadratic time. + >>> mylist = unique([1, 2, 3, 1, 2, 3]) >>> print(sorted(mylist)) [1, 2, 3] @@ -967,17 +1021,6 @@ def unique(seq): >>> mylist = unique(([1, 2], [2, 3], [1, 2])) >>> print(sorted(mylist)) [[1, 2], [2, 3]] - - For best speed, all sequence elements should be hashable. Then - unique() will usually work in linear time. - - If not possible, the sequence elements should enjoy a total - ordering, and if list(s).sort() doesn't raise TypeError it's - assumed that they do enjoy a total ordering. Then unique() will - usually work in O(N*log2(N)) time. - - If that's not possible either, the sequence elements must support - equality-testing. Then unique() will usually work in quadratic time. """ if not seq: @@ -1046,7 +1089,7 @@ def logical_lines(physical_lines, joiner=''.join): class LogicalLines: - """ Wrapper class for the logical_lines method. + """Wrapper class for the :func:`logical_lines` function. Allows us to read all "logical" lines at once from a given file object. """ @@ -1061,9 +1104,10 @@ class LogicalLines: class UniqueList(UserList): """A list which maintains uniqueness. - Uniquing is lazy: rather than being assured on list changes, it is fixed + Uniquing is lazy: rather than being enforced on list changes, it is fixed up on access by those methods which need to act on a unique list to be - correct. That means things like "in" don't have to eat the uniquing time. + correct. That means things like membership tests don't have to eat the + uniquing time. """ def __init__(self, initlist=None) -> None: super().__init__(initlist) @@ -1221,25 +1265,29 @@ def make_path_relative(path) -> str: return path -def silent_intern(x): - """ +def silent_intern(__string: Any) -> str: + """Intern a string without failing. + Perform :mod:`sys.intern` on the passed argument and return the result. If the input is ineligible for interning the original argument is returned and no exception is thrown. """ try: - return sys.intern(x) + return sys.intern(__string) except TypeError: - return x + return __string def cmp(a, b) -> bool: - """A cmp function because one is no longer available in python3.""" + """A cmp function because one is no longer available in Python3.""" return (a > b) - (a < b) def print_time(): """Hack to return a value from Main if can't import Main.""" + # this specifically violates the rule of Util not depending on other + # parts of SCons in order to work around other import-loop issues. + # # pylint: disable=redefined-outer-name,import-outside-toplevel from SCons.Script.Main import print_time return print_time @@ -1278,20 +1326,31 @@ def wait_for_process_to_die(pid) -> None: # From: https://stackoverflow.com/questions/1741972/how-to-use-different-formatters-with-the-same-logging-handler-in-python class DispatchingFormatter(Formatter): + """Logging formatter which dispatches to various formatters.""" def __init__(self, formatters, default_formatter) -> None: self._formatters = formatters self._default_formatter = default_formatter def format(self, record): - formatter = self._formatters.get(record.name, self._default_formatter) + # Search from record's logger up to its parents: + logger = logging.getLogger(record.name) + while logger: + # Check if suitable formatter for current logger exists: + if logger.name in self._formatters: + formatter = self._formatters[logger.name] + break + logger = logger.parent + else: + # If no formatter found, just use default: + formatter = self._default_formatter return formatter.format(record) def sanitize_shell_env(execution_env: dict) -> dict: """Sanitize all values in *execution_env* - The execution environment (typically comes from (env['ENV']) is + The execution environment (typically comes from ``env['ENV']``) is propagated to the shell, and may need to be cleaned first. Args: diff --git a/SCons/Util/envs.py b/SCons/Util/envs.py index 64e728a..db4d65a 100644 --- a/SCons/Util/envs.py +++ b/SCons/Util/envs.py @@ -2,21 +2,26 @@ # # Copyright The SCons Foundation -"""Various SCons utility functions +""" +SCons environment utility functions. Routines for working with environments and construction variables -that don't need the specifics of Environment. +that don't need the specifics of the Environment class. """ import os from types import MethodType, FunctionType -from typing import Union +from typing import Union, Callable, Optional, Any -from .types import is_List, is_Tuple, is_String +from .sctypes import is_List, is_Tuple, is_String def PrependPath( - oldpath, newpath, sep=os.pathsep, delete_existing: bool=True, canonicalize=None + oldpath, + newpath, + sep=os.pathsep, + delete_existing: bool = True, + canonicalize: Optional[Callable] = None, ) -> Union[list, str]: """Prepend *newpath* path elements to *oldpath*. @@ -50,10 +55,10 @@ def PrependPath( if is_String(newpath): newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [newpath] # might be a Dir - else: + elif is_List(newpath) or is_Tuple(newpath): newpaths = newpath + else: + newpaths = [newpath] # might be a Dir if canonicalize: newpaths = list(map(canonicalize, newpaths)) @@ -102,7 +107,11 @@ def PrependPath( def AppendPath( - oldpath, newpath, sep=os.pathsep, delete_existing: bool=True, canonicalize=None + oldpath, + newpath, + sep=os.pathsep, + delete_existing: bool = True, + canonicalize: Optional[Callable] = None, ) -> Union[list, str]: """Append *newpath* path elements to *oldpath*. @@ -136,10 +145,10 @@ def AppendPath( if is_String(newpath): newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [newpath] # might be a Dir - else: + elif is_List(newpath) or is_Tuple(newpath): newpaths = newpath + else: + newpaths = [newpath] # might be a Dir if canonicalize: newpaths = list(map(canonicalize, newpaths)) @@ -187,7 +196,7 @@ def AppendPath( return sep.join(paths) -def AddPathIfNotExists(env_dict, key, path, sep=os.pathsep) -> None: +def AddPathIfNotExists(env_dict, key, path, sep: str = os.pathsep) -> None: """Add a path element to a construction variable. `key` is looked up in `env_dict`, and `path` is added to it if it @@ -229,12 +238,12 @@ class MethodWrapper: a new underlying object being copied (without which we wouldn't need to save that info). """ - def __init__(self, obj, method, name=None) -> None: + def __init__(self, obj: Any, method: Callable, name: Optional[str] = None) -> None: if name is None: name = method.__name__ self.object = obj self.method = method - self.name = name + self.name: str = name setattr(self.object, name, self) def __call__(self, *args, **kwargs): @@ -265,7 +274,7 @@ class MethodWrapper: # is not needed, the remaining bit is now used inline in AddMethod. -def AddMethod(obj, function, name=None) -> None: +def AddMethod(obj, function: Callable, name: Optional[str] = None) -> None: """Add a method to an object. Adds *function* to *obj* if *obj* is a class object. @@ -304,6 +313,8 @@ def AddMethod(obj, function, name=None) -> None: function.__code__, function.__globals__, name, function.__defaults__ ) + method: Union[MethodType, MethodWrapper, Callable] + if hasattr(obj, '__class__') and obj.__class__ is not type: # obj is an instance, so it gets a bound method. if hasattr(obj, "added_methods"): diff --git a/SCons/Util/filelock.py b/SCons/Util/filelock.py index 4b825a6..48a2a39 100644 --- a/SCons/Util/filelock.py +++ b/SCons/Util/filelock.py @@ -141,7 +141,7 @@ class FileLock: def __repr__(self) -> str: """Nicer display if someone repr's the lock class.""" return ( - f"FileLock(" + f"{self.__class__.__name__}(" f"file={self.file!r}, " f"timeout={self.timeout!r}, " f"delay={self.delay!r}, " diff --git a/SCons/Util/hashes.py b/SCons/Util/hashes.py index e14da01..566897a 100644 --- a/SCons/Util/hashes.py +++ b/SCons/Util/hashes.py @@ -2,15 +2,18 @@ # # Copyright The SCons Foundation -"""SCons utility functions +""" +SCons hash utility routines. -Routines for working with hash formats. +Routines for working with content and signature hashes. """ +import functools import hashlib import sys +from typing import Optional, Union -from .types import to_bytes +from .sctypes import to_bytes # Default hash function and format. SCons-internal. @@ -97,9 +100,9 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys) -> None: continue if len(ALLOWED_HASH_FORMATS) == 0: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel SConsEnvironmentError, - ) # pylint: disable=import-outside-toplevel + ) # chain the exception thrown with the most recent error from hashlib. raise SConsEnvironmentError( @@ -159,9 +162,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): if hash_format: hash_format_lower = hash_format.lower() if hash_format_lower not in ALLOWED_HASH_FORMATS: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) # User can select something not supported by their OS but # normally supported by SCons, example, selecting MD5 in an @@ -207,9 +210,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): ) if _HASH_FUNCTION is None: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( f'Hash format "{hash_format_lower}" is not available in your ' @@ -228,9 +231,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): break else: # This is not expected to happen in practice. - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' @@ -270,9 +273,9 @@ def _get_hash_object(hash_format, hashlib_used=hashlib, sys_used=sys): """ if hash_format is None: if _HASH_FUNCTION is None: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( 'There is no default hash function. Did you call ' @@ -334,6 +337,10 @@ def hash_file_signature(fname, chunksize: int=65536, hash_format=None): if not blck: break m.update(to_bytes(blck)) + # TODO: can use this when base is Python 3.8+ + # while (blk := f.read(chunksize)) != b'': + # m.update(to_bytes(blk)) + return m.hexdigest() diff --git a/SCons/Util/sctypes.py b/SCons/Util/sctypes.py new file mode 100644 index 0000000..53fcc56 --- /dev/null +++ b/SCons/Util/sctypes.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + +"""Various SCons utility functions + +Routines which check types and do type conversions. +""" + +import os +import pprint +import re +from typing import Optional + +from collections import UserDict, UserList, UserString, deque +from collections.abc import MappingView, Iterable + +# Functions for deciding if things are like various types, mainly to +# handle UserDict, UserList and UserString like their underlying types. +# +# Yes, all of this manual testing breaks polymorphism, and the real +# Pythonic way to do all of this would be to just try it and handle the +# exception, but handling the exception when it's not the right type is +# often too slow. + +# A trick is used to speed up these functions. Default arguments are +# used to take a snapshot of the global functions and constants used +# by these functions. This transforms accesses to global variables into +# local variable accesses (i.e. LOAD_FAST instead of LOAD_GLOBAL). +# Since checkers dislike this, it's now annotated for pylint, to flag +# (mostly for other readers of this code) we're doing this intentionally. +# TODO: experts affirm this is still faster, but maybe check if worth it? + +DictTypes = (dict, UserDict) +ListTypes = (list, UserList, deque) + +# With Python 3, there are view types that are sequences. Other interesting +# sequences are range and bytearray. What we don't want is strings: while +# they are iterable sequences, in SCons usage iterating over a string is +# almost never what we want. So basically iterable-but-not-string: +SequenceTypes = (list, tuple, deque, UserList, MappingView) + +# Note that profiling data shows a speed-up when comparing +# explicitly with str instead of simply comparing +# with basestring. (at least on Python 2.5.1) +# TODO: PY3 check this benchmarking is still correct. +StringTypes = (str, UserString) + +# Empirically, it is faster to check explicitly for str than for basestring. +BaseStringTypes = str + + +def is_Dict( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, DictTypes=DictTypes +) -> bool: + """Check if object is a dict.""" + return isinstance(obj, DictTypes) + + +def is_List( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, ListTypes=ListTypes +) -> bool: + """Check if object is a list.""" + return isinstance(obj, ListTypes) + + +def is_Sequence( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, SequenceTypes=SequenceTypes +) -> bool: + """Check if object is a sequence.""" + return isinstance(obj, SequenceTypes) + + +def is_Tuple( # pylint: disable=redefined-builtin + obj, isinstance=isinstance, tuple=tuple +) -> bool: + """Check if object is a tuple.""" + return isinstance(obj, tuple) + + +def is_String( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, StringTypes=StringTypes +) -> bool: + """Check if object is a string.""" + return isinstance(obj, StringTypes) + + +def is_Scalar( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, StringTypes=StringTypes, Iterable=Iterable, +) -> bool: + """Check if object is a scalar: not a container or iterable.""" + # Profiling shows that there is an impressive speed-up of 2x + # when explicitly checking for strings instead of just not + # sequence when the argument (i.e. obj) is already a string. + # But, if obj is a not string then it is twice as fast to + # check only for 'not sequence'. The following code therefore + # assumes that the obj argument is a string most of the time. + # Update: now using collections.abc.Iterable for the 2nd check. + # Note: None is considered a "scalar" for this check, which is correct + # for the usage in SCons.Environment._add_cppdefines. + return isinstance(obj, StringTypes) or not isinstance(obj, Iterable) + + +# From Dinu C. Gherman, +# Python Cookbook, second edition, recipe 6.17, p. 277. +# Also: https://code.activestate.com/recipes/68205 +# ASPN: Python Cookbook: Null Object Design Pattern + + +class Null: + """Null objects always and reliably 'do nothing'.""" + + def __new__(cls, *args, **kwargs): + if '_instance' not in vars(cls): + cls._instance = super().__new__(cls, *args, **kwargs) + return cls._instance + + def __init__(self, *args, **kwargs) -> None: + pass + + def __call__(self, *args, **kwargs): + return self + + def __repr__(self) -> str: + return f"Null(0x{id(self):08X})" + + def __bool__(self) -> bool: + return False + + def __getattr__(self, name): + return self + + def __setattr__(self, name, value): + return self + + def __delattr__(self, name): + return self + + +class NullSeq(Null): + """A Null object that can also be iterated over.""" + + def __len__(self) -> int: + return 0 + + def __iter__(self): + return iter(()) + + def __getitem__(self, i): + return self + + def __delitem__(self, i): + return self + + def __setitem__(self, i, v): + return self + + +def to_bytes(s) -> bytes: + """Convert object to bytes.""" + if s is None: + return b'None' + if isinstance(s, (bytes, bytearray)): + # if already bytes return. + return s + return bytes(s, 'utf-8') + + +def to_str(s) -> str: + """Convert object to string.""" + if s is None: + return 'None' + if is_String(s): + return s + return str(s, 'utf-8') + + +# Generic convert-to-string functions. The wrapper +# to_String_for_signature() will use a for_signature() method if the +# specified object has one. + + +def to_String( # pylint: disable=redefined-outer-name,redefined-builtin + obj, + isinstance=isinstance, + str=str, + UserString=UserString, + BaseStringTypes=BaseStringTypes, +) -> str: + """Return a string version of obj.""" + if isinstance(obj, BaseStringTypes): + # Early out when already a string! + return obj + + if isinstance(obj, UserString): + # obj.data can only be a regular string. Please see the UserString initializer. + return obj.data + + return str(obj) + + +def to_String_for_subst( # pylint: disable=redefined-outer-name,redefined-builtin + obj, + isinstance=isinstance, + str=str, + BaseStringTypes=BaseStringTypes, + SequenceTypes=SequenceTypes, + UserString=UserString, +) -> str: + """Return a string version of obj for subst usage.""" + # Note that the test cases are sorted by order of probability. + if isinstance(obj, BaseStringTypes): + return obj + + if isinstance(obj, SequenceTypes): + return ' '.join([to_String_for_subst(e) for e in obj]) + + if isinstance(obj, UserString): + # obj.data can only a regular string. Please see the UserString initializer. + return obj.data + + return str(obj) + + +def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-builtin + obj, to_String_for_subst=to_String_for_subst, AttributeError=AttributeError, +) -> str: + """Return a string version of obj for signature usage. + + Like :func:`to_String_for_subst` but has special handling for + scons objects that have a :meth:`for_signature` method, and for dicts. + """ + try: + f = obj.for_signature + except AttributeError: + if isinstance(obj, dict): + # pprint will output dictionary in key sorted order + # with py3.5 the order was randomized. Depending on dict order + # which was undefined until py3.6 (where it's by insertion order) + # was not wise. + # TODO: Change code when floor is raised to PY36 + return pprint.pformat(obj, width=1000000) + return to_String_for_subst(obj) + return f() + + +def get_env_bool(env, name: str, default: bool=False) -> bool: + """Convert a construction variable to bool. + + If the value of *name* in dict-like object *env* is 'true', 'yes', + 'y', 'on' (case insensitive) or anything convertible to int that + yields non-zero, return ``True``; if 'false', 'no', 'n', 'off' + (case insensitive) or a number that converts to integer zero return + ``False``. Otherwise, or if *name* is not found, return the value + of *default*. + + Args: + env: construction environment, or any dict-like object. + name: name of the variable. + default: value to return if *name* not in *env* or cannot + be converted (default: False). + """ + try: + var = env[name] + except KeyError: + return default + try: + return bool(int(var)) + except ValueError: + if str(var).lower() in ('true', 'yes', 'y', 'on'): + return True + + if str(var).lower() in ('false', 'no', 'n', 'off'): + return False + + return default + + +def get_os_env_bool(name: str, default: bool=False) -> bool: + """Convert an external environment variable to boolean. + + Like :func:`get_env_bool`, but uses :attr:`os.environ` as the lookup dict. + """ + return get_env_bool(os.environ, name, default) + + +_get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') + + +def get_environment_var(varstr) -> Optional[str]: + """Return undecorated construction variable string. + + Determine if *varstr* looks like a reference + to a single environment variable, like ``"$FOO"`` or ``"${FOO}"``. + If so, return that variable with no decorations, like ``"FOO"``. + If not, return ``None``. + """ + mo = _get_env_var.match(to_String(varstr)) + if mo: + var = mo.group(1) + if var[0] == '{': + return var[1:-1] + return var + + return None + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/stats.py b/SCons/Util/stats.py index 22135e3..ce820e6 100644 --- a/SCons/Util/stats.py +++ b/SCons/Util/stats.py @@ -22,17 +22,26 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -This package provides a way to gather various statistics during a SCons run and dump that info in several formats +SCons statistics routines. -Additionally, it probably makes sense to do stderr/stdout output of those statistics here as well +This package provides a way to gather various statistics during an SCons +run and dump that info in several formats -There are basically two types of stats. -1. Timer (start/stop/time) for specific event. These events can be hierarchical. So you can record the children events of some parent. - Think program compile could contain the total Program builder time, which could include linking, and stripping the executable -2. Counter. Counting the number of events and/or objects created. This would likely only be reported at the end of a given SCons run, - though it might be useful to query during a run. +Additionally, it probably makes sense to do stderr/stdout output of +those statistics here as well + +There are basically two types of stats: +1. Timer (start/stop/time) for specific event. These events can be + hierarchical. So you can record the children events of some parent. + Think program compile could contain the total Program builder time, + which could include linking, and stripping the executable + +2. Counter. Counting the number of events and/or objects created. This + would likely only be reported at the end of a given SCons run, + though it might be useful to query during a run. """ + from abc import ABC import platform @@ -47,13 +56,10 @@ ENABLE_JSON = False JSON_OUTPUT_FILE = 'scons_stats.json' def add_stat_type(name, stat_object): - """ - Add a statistic type to the global collection - """ + """Add a statistic type to the global collection""" if name in all_stats: raise UserWarning(f'Stat type {name} already exists') - else: - all_stats[name] = stat_object + all_stats[name] = stat_object class Stats(ABC): @@ -108,8 +114,8 @@ class CountStats(Stats): fmt2 = ''.join(pre + [' %7d'] * l + post) labels = self.labels[:l] labels.append(("", "Class")) - self.outfp.write(fmt1 % tuple([x[0] for x in labels])) - self.outfp.write(fmt1 % tuple([x[1] for x in labels])) + self.outfp.write(fmt1 % tuple(x[0] for x in labels)) + self.outfp.write(fmt1 % tuple(x[1] for x in labels)) for k in sorted(self.stats_table.keys()): r = self.stats_table[k][:l] + [k] self.outfp.write(fmt2 % tuple(r)) @@ -160,9 +166,12 @@ def write_scons_stats_file(): """ # Have to import where used to avoid import loop - from SCons.Script import BUILD_TARGETS, COMMAND_LINE_TARGETS, ARGUMENTS, \ - ARGLIST # [import-outside-toplevel] - + from SCons.Script import ( # pylint: disable=import-outside-toplevel + BUILD_TARGETS, + COMMAND_LINE_TARGETS, + ARGUMENTS, + ARGLIST, + ) # print(f"DUMPING JSON FILE: {JSON_OUTPUT_FILE}") json_structure = {} if count_stats.enabled: diff --git a/SCons/Util/types.py b/SCons/Util/types.py deleted file mode 100644 index b2bc040..0000000 --- a/SCons/Util/types.py +++ /dev/null @@ -1,315 +0,0 @@ -# SPDX-License-Identifier: MIT -# -# Copyright The SCons Foundation - -"""Various SCons utility functions - -Routines which check types and do type conversions. -""" - -import os -import pprint -import re -from typing import Optional - -from collections import UserDict, UserList, UserString, deque -from collections.abc import MappingView, Iterable - -# Functions for deciding if things are like various types, mainly to -# handle UserDict, UserList and UserString like their underlying types. -# -# Yes, all of this manual testing breaks polymorphism, and the real -# Pythonic way to do all of this would be to just try it and handle the -# exception, but handling the exception when it's not the right type is -# often too slow. - -# A trick is used to speed up these functions. Default arguments are -# used to take a snapshot of the global functions and constants used -# by these functions. This transforms accesses to global variables into -# local variable accesses (i.e. LOAD_FAST instead of LOAD_GLOBAL). -# Since checkers dislike this, it's now annotated for pylint, to flag -# (mostly for other readers of this code) we're doing this intentionally. -# TODO: experts affirm this is still faster, but maybe check if worth it? - -DictTypes = (dict, UserDict) -ListTypes = (list, UserList, deque) - -# With Python 3, there are view types that are sequences. Other interesting -# sequences are range and bytearray. What we don't want is strings: while -# they are iterable sequences, in SCons usage iterating over a string is -# almost never what we want. So basically iterable-but-not-string: -SequenceTypes = (list, tuple, deque, UserList, MappingView) - -# Note that profiling data shows a speed-up when comparing -# explicitly with str instead of simply comparing -# with basestring. (at least on Python 2.5.1) -# TODO: PY3 check this benchmarking is still correct. -StringTypes = (str, UserString) - -# Empirically, it is faster to check explicitly for str than for basestring. -BaseStringTypes = str - - -def is_Dict( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, DictTypes=DictTypes -) -> bool: - """Check if object is a dict.""" - return isinstance(obj, DictTypes) - - -def is_List( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, ListTypes=ListTypes -) -> bool: - """Check if object is a list.""" - return isinstance(obj, ListTypes) - - -def is_Sequence( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, SequenceTypes=SequenceTypes -) -> bool: - """Check if object is a sequence.""" - return isinstance(obj, SequenceTypes) - - -def is_Tuple( # pylint: disable=redefined-builtin - obj, isinstance=isinstance, tuple=tuple -) -> bool: - """Check if object is a tuple.""" - return isinstance(obj, tuple) - - -def is_String( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, StringTypes=StringTypes -) -> bool: - """Check if object is a string.""" - return isinstance(obj, StringTypes) - - -def is_Scalar( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, StringTypes=StringTypes, Iterable=Iterable, -) -> bool: - """Check if object is a scalar: not a container or iterable.""" - # Profiling shows that there is an impressive speed-up of 2x - # when explicitly checking for strings instead of just not - # sequence when the argument (i.e. obj) is already a string. - # But, if obj is a not string then it is twice as fast to - # check only for 'not sequence'. The following code therefore - # assumes that the obj argument is a string most of the time. - # Update: now using collections.abc.Iterable for the 2nd check. - # Note: None is considered a "scalar" for this check, which is correct - # for the usage in SCons.Environment._add_cppdefines. - return isinstance(obj, StringTypes) or not isinstance(obj, Iterable) - - -# From Dinu C. Gherman, -# Python Cookbook, second edition, recipe 6.17, p. 277. -# Also: https://code.activestate.com/recipes/68205 -# ASPN: Python Cookbook: Null Object Design Pattern - - -class Null: - """Null objects always and reliably 'do nothing'.""" - - def __new__(cls, *args, **kwargs): - if '_instance' not in vars(cls): - cls._instance = super().__new__(cls, *args, **kwargs) - return cls._instance - - def __init__(self, *args, **kwargs) -> None: - pass - - def __call__(self, *args, **kwargs): - return self - - def __repr__(self) -> str: - return f"Null(0x{id(self):08X})" - - def __bool__(self) -> bool: - return False - - def __getattr__(self, name): - return self - - def __setattr__(self, name, value): - return self - - def __delattr__(self, name): - return self - - -class NullSeq(Null): - """A Null object that can also be iterated over.""" - - def __len__(self) -> int: - return 0 - - def __iter__(self): - return iter(()) - - def __getitem__(self, i): - return self - - def __delitem__(self, i): - return self - - def __setitem__(self, i, v): - return self - - -def to_bytes(s) -> bytes: - """Convert object to bytes.""" - if s is None: - return b'None' - if isinstance(s, (bytes, bytearray)): - # if already bytes return. - return s - return bytes(s, 'utf-8') - - -def to_str(s) -> str: - """Convert object to string.""" - if s is None: - return 'None' - if is_String(s): - return s - return str(s, 'utf-8') - - -# Generic convert-to-string functions. The wrapper -# to_String_for_signature() will use a for_signature() method if the -# specified object has one. - - -def to_String( # pylint: disable=redefined-outer-name,redefined-builtin - obj, - isinstance=isinstance, - str=str, - UserString=UserString, - BaseStringTypes=BaseStringTypes, -) -> str: - """Return a string version of obj.""" - if isinstance(obj, BaseStringTypes): - # Early out when already a string! - return obj - - if isinstance(obj, UserString): - # obj.data can only be a regular string. Please see the UserString initializer. - return obj.data - - return str(obj) - - -def to_String_for_subst( # pylint: disable=redefined-outer-name,redefined-builtin - obj, - isinstance=isinstance, - str=str, - BaseStringTypes=BaseStringTypes, - SequenceTypes=SequenceTypes, - UserString=UserString, -) -> str: - """Return a string version of obj for subst usage.""" - # Note that the test cases are sorted by order of probability. - if isinstance(obj, BaseStringTypes): - return obj - - if isinstance(obj, SequenceTypes): - return ' '.join([to_String_for_subst(e) for e in obj]) - - if isinstance(obj, UserString): - # obj.data can only a regular string. Please see the UserString initializer. - return obj.data - - return str(obj) - - -def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-builtin - obj, to_String_for_subst=to_String_for_subst, AttributeError=AttributeError, -) -> str: - """Return a string version of obj for signature usage. - - Like :func:`to_String_for_subst` but has special handling for - scons objects that have a :meth:`for_signature` method, and for dicts. - """ - try: - f = obj.for_signature - except AttributeError: - if isinstance(obj, dict): - # pprint will output dictionary in key sorted order - # with py3.5 the order was randomized. Depending on dict order - # which was undefined until py3.6 (where it's by insertion order) - # was not wise. - # TODO: Change code when floor is raised to PY36 - return pprint.pformat(obj, width=1000000) - return to_String_for_subst(obj) - else: - return f() - - -def get_env_bool(env, name, default: bool=False) -> bool: - """Convert a construction variable to bool. - - If the value of *name* in *env* is 'true', 'yes', 'y', 'on' (case - insensitive) or anything convertible to int that yields non-zero then - return ``True``; if 'false', 'no', 'n', 'off' (case insensitive) - or a number that converts to integer zero return ``False``. - Otherwise, return `default`. - - Args: - env: construction environment, or any dict-like object - name: name of the variable - default: value to return if *name* not in *env* or cannot - be converted (default: False) - - Returns: - the "truthiness" of `name` - """ - try: - var = env[name] - except KeyError: - return default - try: - return bool(int(var)) - except ValueError: - if str(var).lower() in ('true', 'yes', 'y', 'on'): - return True - - if str(var).lower() in ('false', 'no', 'n', 'off'): - return False - - return default - - -def get_os_env_bool(name, default: bool=False) -> bool: - """Convert an environment variable to bool. - - Conversion is the same as for :func:`get_env_bool`. - """ - return get_env_bool(os.environ, name, default) - - -_get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') - - -def get_environment_var(varstr) -> Optional[str]: - """Return undecorated construction variable string. - - Determine if `varstr` looks like a reference - to a single environment variable, like `"$FOO"` or `"${FOO}"`. - If so, return that variable with no decorations, like `"FOO"`. - If not, return `None`. - """ - mo = _get_env_var.match(to_String(varstr)) - if mo: - var = mo.group(1) - if var[0] == '{': - return var[1:-1] - return var - - return None - - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py deleted file mode 100644 index 54cb658..0000000 --- a/SCons/UtilTests.py +++ /dev/null @@ -1,1215 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import functools -import io -import os -import sys -import unittest -import unittest.mock -import hashlib -import warnings -from collections import UserDict, UserList, UserString, namedtuple - -import TestCmd - -import SCons.Errors -import SCons.compat -from SCons.Util import ( - ALLOWED_HASH_FORMATS, - AddPathIfNotExists, - AppendPath, - CLVar, - LogicalLines, - NodeList, - PrependPath, - Proxy, - Selector, - WhereIs, - adjustixes, - containsAll, - containsAny, - containsOnly, - dictify, - display, - flatten, - get_env_bool, - get_environment_var, - get_native_path, - get_os_env_bool, - hash_collect, - hash_signature, - is_Dict, - is_List, - is_String, - is_Tuple, - print_tree, - render_tree, - set_hash_format, - silent_intern, - splitext, - to_String, - to_bytes, - to_str, -) -from SCons.Util.hashes import ( - _attempt_init_of_python_3_9_hash_object, - _attempt_get_hash_function, - _get_hash_object, - _set_allowed_viable_default_hashes, -) - -# These Util classes have no unit tests. Some don't make sense to test? -# DisplayEngine, Delegate, MethodWrapper, UniqueList, Unbuffered, Null, NullSeq - - -class OutBuffer: - def __init__(self) -> None: - self.buffer = "" - - def write(self, str) -> None: - self.buffer = self.buffer + str - - -class dictifyTestCase(unittest.TestCase): - def test_dictify(self) -> None: - """Test the dictify() function""" - r = dictify(['a', 'b', 'c'], [1, 2, 3]) - assert r == {'a': 1, 'b': 2, 'c': 3}, r - - r = {} - dictify(['a'], [1], r) - dictify(['b'], [2], r) - dictify(['c'], [3], r) - assert r == {'a': 1, 'b': 2, 'c': 3}, r - - -class UtilTestCase(unittest.TestCase): - def test_splitext(self) -> None: - assert splitext('foo') == ('foo', '') - assert splitext('foo.bar') == ('foo', '.bar') - assert splitext(os.path.join('foo.bar', 'blat')) == (os.path.join('foo.bar', 'blat'), '') - - class Node: - def __init__(self, name, children=[]) -> None: - self.children = children - self.name = name - self.nocache = None - - def __str__(self) -> str: - return self.name - - def exists(self) -> bool: - return True - - def rexists(self) -> bool: - return True - - def has_builder(self) -> bool: - return True - - def has_explicit_builder(self) -> bool: - return True - - def side_effect(self) -> bool: - return True - - def precious(self) -> bool: - return True - - def always_build(self) -> bool: - return True - - def is_up_to_date(self) -> bool: - return True - - def noclean(self) -> bool: - return True - - def tree_case_1(self): - """Fixture for the render_tree() and print_tree() tests.""" - windows_h = self.Node("windows.h") - stdlib_h = self.Node("stdlib.h") - stdio_h = self.Node("stdio.h") - bar_c = self.Node("bar.c", [stdlib_h, windows_h]) - bar_o = self.Node("bar.o", [bar_c]) - foo_c = self.Node("foo.c", [stdio_h]) - foo_o = self.Node("foo.o", [foo_c]) - foo = self.Node("foo", [foo_o, bar_o]) - - expect = """\ -+-foo - +-foo.o - | +-foo.c - | +-stdio.h - +-bar.o - +-bar.c - +-stdlib.h - +-windows.h -""" - - lines = expect.split('\n')[:-1] - lines = ['[E BSPACN ]' + l for l in lines] - withtags = '\n'.join(lines) + '\n' - - return foo, expect, withtags - - def tree_case_2(self, prune: int=1): - """Fixture for the render_tree() and print_tree() tests.""" - - types_h = self.Node('types.h') - malloc_h = self.Node('malloc.h') - stdlib_h = self.Node('stdlib.h', [types_h, malloc_h]) - bar_h = self.Node('bar.h', [stdlib_h]) - blat_h = self.Node('blat.h', [stdlib_h]) - blat_c = self.Node('blat.c', [blat_h, bar_h]) - blat_o = self.Node('blat.o', [blat_c]) - - expect = """\ -+-blat.o - +-blat.c - +-blat.h - | +-stdlib.h - | +-types.h - | +-malloc.h - +-bar.h -""" - if prune: - expect += """ +-[stdlib.h] -""" - else: - expect += """ +-stdlib.h - +-types.h - +-malloc.h -""" - - lines = expect.split('\n')[:-1] - lines = ['[E BSPACN ]' + l for l in lines] - withtags = '\n'.join(lines) + '\n' - - return blat_o, expect, withtags - - def test_render_tree(self) -> None: - """Test the render_tree() function""" - - def get_children(node): - return node.children - - node, expect, withtags = self.tree_case_1() - actual = render_tree(node, get_children) - assert expect == actual, (expect, actual) - - node, expect, withtags = self.tree_case_2() - actual = render_tree(node, get_children, 1) - assert expect == actual, (expect, actual) - - # Ensure that we can call render_tree on the same Node - # again. This wasn't possible in version 2.4.1 and earlier - # due to a bug in render_tree (visited was set to {} as default - # parameter) - actual = render_tree(node, get_children, 1) - assert expect == actual, (expect, actual) - - def test_print_tree(self) -> None: - """Test the print_tree() function""" - - def get_children(node): - return node.children - - save_stdout = sys.stdout - - try: - node, expect, withtags = self.tree_case_1() - - IOStream = io.StringIO - sys.stdout = IOStream() - print_tree(node, get_children) - actual = sys.stdout.getvalue() - assert expect == actual, (expect, actual) - - sys.stdout = IOStream() - print_tree(node, get_children, showtags=1) - actual = sys.stdout.getvalue() - assert withtags == actual, (withtags, actual) - - # Test that explicitly setting prune to zero works - # the same as the default (see above) - node, expect, withtags = self.tree_case_2(prune=0) - - sys.stdout = IOStream() - print_tree(node, get_children, 0) - actual = sys.stdout.getvalue() - assert expect == actual, (expect, actual) - - sys.stdout = IOStream() - print_tree(node, get_children, 0, showtags=1) - actual = sys.stdout.getvalue() - assert withtags == actual, (withtags, actual) - - # Test output with prune=1 - node, expect, withtags = self.tree_case_2(prune=1) - - sys.stdout = IOStream() - print_tree(node, get_children, 1) - actual = sys.stdout.getvalue() - assert expect == actual, (expect, actual) - - # Ensure that we can call print_tree on the same Node - # again. This wasn't possible in version 2.4.1 and earlier - # due to a bug in print_tree (visited was set to {} as default - # parameter) - sys.stdout = IOStream() - print_tree(node, get_children, 1) - actual = sys.stdout.getvalue() - assert expect == actual, (expect, actual) - - sys.stdout = IOStream() - print_tree(node, get_children, 1, showtags=1) - actual = sys.stdout.getvalue() - assert withtags == actual, (withtags, actual) - finally: - sys.stdout = save_stdout - - def test_is_Dict(self) -> None: - assert is_Dict({}) - assert is_Dict(UserDict()) - try: - class mydict(dict): - pass - except TypeError: - pass - else: - assert is_Dict(mydict({})) - assert not is_Dict([]) - assert not is_Dict(()) - assert not is_Dict("") - - - def test_is_List(self) -> None: - assert is_List([]) - assert is_List(UserList()) - try: - class mylist(list): - pass - except TypeError: - pass - else: - assert is_List(mylist([])) - assert not is_List(()) - assert not is_List({}) - assert not is_List("") - - def test_is_String(self) -> None: - assert is_String("") - assert is_String(UserString('')) - try: - class mystr(str): - pass - except TypeError: - pass - else: - assert is_String(mystr('')) - assert not is_String({}) - assert not is_String([]) - assert not is_String(()) - - def test_is_Tuple(self) -> None: - assert is_Tuple(()) - try: - class mytuple(tuple): - pass - except TypeError: - pass - else: - assert is_Tuple(mytuple(())) - assert not is_Tuple([]) - assert not is_Tuple({}) - assert not is_Tuple("") - - def test_to_Bytes(self) -> None: - """ Test the to_Bytes method""" - self.assertEqual(to_bytes('Hello'), - bytearray('Hello', 'utf-8'), - "Check that to_bytes creates byte array when presented with non byte string.") - - def test_to_String(self) -> None: - """Test the to_String() method.""" - assert to_String(1) == "1", to_String(1) - assert to_String([1, 2, 3]) == str([1, 2, 3]), to_String([1, 2, 3]) - assert to_String("foo") == "foo", to_String("foo") - assert to_String(None) == 'None' - # test low level string converters too - assert to_str(None) == 'None' - assert to_bytes(None) == b'None' - - s1 = UserString('blah') - assert to_String(s1) == s1, s1 - assert to_String(s1) == 'blah', s1 - - class Derived(UserString): - pass - - s2 = Derived('foo') - assert to_String(s2) == s2, s2 - assert to_String(s2) == 'foo', s2 - - - def test_WhereIs(self) -> None: - test = TestCmd.TestCmd(workdir='') - - sub1_xxx_exe = test.workpath('sub1', 'xxx.exe') - sub2_xxx_exe = test.workpath('sub2', 'xxx.exe') - sub3_xxx_exe = test.workpath('sub3', 'xxx.exe') - sub4_xxx_exe = test.workpath('sub4', 'xxx.exe') - - test.subdir('subdir', 'sub1', 'sub2', 'sub3', 'sub4') - - if sys.platform != 'win32': - test.write(sub1_xxx_exe, "\n") - - os.mkdir(sub2_xxx_exe) - - test.write(sub3_xxx_exe, "\n") - os.chmod(sub3_xxx_exe, 0o777) - - test.write(sub4_xxx_exe, "\n") - os.chmod(sub4_xxx_exe, 0o777) - - env_path = os.environ['PATH'] - - try: - pathdirs_1234 = [test.workpath('sub1'), - test.workpath('sub2'), - test.workpath('sub3'), - test.workpath('sub4'), - ] + env_path.split(os.pathsep) - - pathdirs_1243 = [test.workpath('sub1'), - test.workpath('sub2'), - test.workpath('sub4'), - test.workpath('sub3'), - ] + env_path.split(os.pathsep) - - os.environ['PATH'] = os.pathsep.join(pathdirs_1234) - wi = WhereIs('xxx.exe') - assert wi == test.workpath(sub3_xxx_exe), wi - wi = WhereIs('xxx.exe', pathdirs_1243) - assert wi == test.workpath(sub4_xxx_exe), wi - wi = WhereIs('xxx.exe', os.pathsep.join(pathdirs_1243)) - assert wi == test.workpath(sub4_xxx_exe), wi - - wi = WhereIs('xxx.exe', reject=sub3_xxx_exe) - assert wi == test.workpath(sub4_xxx_exe), wi - wi = WhereIs('xxx.exe', pathdirs_1243, reject=sub3_xxx_exe) - assert wi == test.workpath(sub4_xxx_exe), wi - - os.environ['PATH'] = os.pathsep.join(pathdirs_1243) - wi = WhereIs('xxx.exe') - assert wi == test.workpath(sub4_xxx_exe), wi - wi = WhereIs('xxx.exe', pathdirs_1234) - assert wi == test.workpath(sub3_xxx_exe), wi - wi = WhereIs('xxx.exe', os.pathsep.join(pathdirs_1234)) - assert wi == test.workpath(sub3_xxx_exe), wi - - if sys.platform == 'win32': - wi = WhereIs('xxx', pathext='') - assert wi is None, wi - - wi = WhereIs('xxx', pathext='.exe') - assert wi == test.workpath(sub4_xxx_exe), wi - - wi = WhereIs('xxx', path=pathdirs_1234, pathext='.BAT;.EXE') - assert wi.lower() == test.workpath(sub3_xxx_exe).lower(), wi - - # Test that we return a normalized path even when - # the path contains forward slashes. - forward_slash = test.workpath('') + '/sub3' - wi = WhereIs('xxx', path=forward_slash, pathext='.EXE') - assert wi.lower() == test.workpath(sub3_xxx_exe).lower(), wi - - del os.environ['PATH'] - wi = WhereIs('xxx.exe') - assert wi is None, wi - - finally: - os.environ['PATH'] = env_path - - def test_get_env_var(self) -> None: - """Testing get_environment_var().""" - assert get_environment_var("$FOO") == "FOO", get_environment_var("$FOO") - assert get_environment_var("${BAR}") == "BAR", get_environment_var("${BAR}") - assert get_environment_var("$FOO_BAR1234") == "FOO_BAR1234", get_environment_var("$FOO_BAR1234") - assert get_environment_var("${BAR_FOO1234}") == "BAR_FOO1234", get_environment_var("${BAR_FOO1234}") - assert get_environment_var("${BAR}FOO") is None, get_environment_var("${BAR}FOO") - assert get_environment_var("$BAR ") is None, get_environment_var("$BAR ") - assert get_environment_var("FOO$BAR") is None, get_environment_var("FOO$BAR") - assert get_environment_var("$FOO[0]") is None, get_environment_var("$FOO[0]") - assert get_environment_var("${some('complex expression')}") is None, get_environment_var( - "${some('complex expression')}") - - def test_Proxy(self) -> None: - """Test generic Proxy class.""" - - class Subject: - def foo(self) -> int: - return 1 - - def bar(self) -> int: - return 2 - - s = Subject() - s.baz = 3 - - class ProxyTest(Proxy): - def bar(self) -> int: - return 4 - - p = ProxyTest(s) - - assert p.foo() == 1, p.foo() - assert p.bar() == 4, p.bar() - assert p.baz == 3, p.baz - - p.baz = 5 - s.baz = 6 - - assert p.baz == 5, p.baz - assert p.get() == s, p.get() - - def test_display(self) -> None: - old_stdout = sys.stdout - sys.stdout = OutBuffer() - display("line1") - display.set_mode(0) - display("line2") - display.set_mode(1) - display("line3") - display("line4\n", append_newline=0) - display.set_mode(0) - display("dont print1") - display("dont print2\n", append_newline=0) - display.set_mode(1) - assert sys.stdout.buffer == "line1\nline3\nline4\n" - sys.stdout = old_stdout - - def test_get_native_path(self) -> None: - """Test the get_native_path() function.""" - import tempfile - f, filename = tempfile.mkstemp(text=True) - os.close(f) - data = '1234567890 ' + filename - try: - with open(filename, 'w') as f: - f.write(data) - with open(get_native_path(filename), 'r') as f: - assert f.read() == data - finally: - try: - os.unlink(filename) - except OSError: - pass - - def test_PrependPath(self) -> None: - """Test prepending to a path""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - p2 = r'C:\mydir\num\one;C:\mydir\num\two' - # have to include the pathsep here so that the test will work on UNIX too. - p1 = PrependPath(p1, r'C:\dir\num\two', sep=';') - p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') - assert p1 == r'C:\dir\num\three;C:\dir\num\two;C:\dir\num\one', p1 - - p2 = PrependPath(p2, r'C:\mydir\num\three', sep=';') - p2 = PrependPath(p2, r'C:\mydir\num\one', sep=';') - assert p2 == r'C:\mydir\num\one;C:\mydir\num\three;C:\mydir\num\two', p2 - - # check (only) first one is kept if there are dupes in new - p3 = r'C:\dir\num\one' - p3 = PrependPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') - assert p3 == r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one', p3 - - def test_AppendPath(self) -> None: - """Test appending to a path.""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - p2 = r'C:\mydir\num\one;C:\mydir\num\two' - # have to include the pathsep here so that the test will work on UNIX too. - p1 = AppendPath(p1, r'C:\dir\num\two', sep=';') - p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') - assert p1 == r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three', p1 - - p2 = AppendPath(p2, r'C:\mydir\num\three', sep=';') - p2 = AppendPath(p2, r'C:\mydir\num\one', sep=';') - assert p2 == r'C:\mydir\num\two;C:\mydir\num\three;C:\mydir\num\one', p2 - - # check (only) last one is kept if there are dupes in new - p3 = r'C:\dir\num\one' - p3 = AppendPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') - assert p3 == r'C:\dir\num\one;C:\dir\num\three;C:\dir\num\two', p3 - - def test_PrependPathPreserveOld(self) -> None: - """Test prepending to a path while preserving old paths""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - # have to include the pathsep here so that the test will work on UNIX too. - p1 = PrependPath(p1, r'C:\dir\num\two', sep=';', delete_existing=0) - p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') - assert p1 == r'C:\dir\num\three;C:\dir\num\one;C:\dir\num\two', p1 - - def test_AppendPathPreserveOld(self) -> None: - """Test appending to a path while preserving old paths""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - # have to include the pathsep here so that the test will work on UNIX too. - p1 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=0) - p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') - assert p1 == r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three', p1 - - def test_addPathIfNotExists(self) -> None: - """Test the AddPathIfNotExists() function""" - env_dict = {'FOO': os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), - 'BAR': os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), - 'BLAT': [os.path.normpath('/foo/bar'), - os.path.normpath('/baz/blat')]} - AddPathIfNotExists(env_dict, 'FOO', os.path.normpath('/foo/bar')) - AddPathIfNotExists(env_dict, 'BAR', os.path.normpath('/bar/foo')) - AddPathIfNotExists(env_dict, 'BAZ', os.path.normpath('/foo/baz')) - AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/blat')) - AddPathIfNotExists(env_dict, 'BLAT', os.path.normpath('/baz/foo')) - - assert env_dict['FOO'] == os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), env_dict['FOO'] - assert env_dict['BAR'] == os.path.normpath('/bar/foo') + os.pathsep + \ - os.path.normpath('/foo/bar') + os.pathsep + \ - os.path.normpath('/baz/blat'), env_dict['BAR'] - assert env_dict['BAZ'] == os.path.normpath('/foo/baz'), env_dict['BAZ'] - assert env_dict['BLAT'] == [os.path.normpath('/baz/foo'), - os.path.normpath('/foo/bar'), - os.path.normpath('/baz/blat')], env_dict['BLAT'] - - def test_CLVar(self) -> None: - """Test the command-line construction variable class""" - - # the default value should be an empty list - d = CLVar() - assert isinstance(d, CLVar), type(d) - assert d.data == [], d.data - assert str(d) == '', str(d) - - # input to CLVar is a string - should be split - f = CLVar('aa bb') - - r = f + 'cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ' cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', ' cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', ' cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - # input to CLVar is a list of one string, should not be split - f = CLVar(['aa bb']) - - r = f + 'cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ' cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', 'cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', ' cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa bb', ' cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - # input to CLVar is a list of strings - f = CLVar(['aa', 'bb']) - - r = f + 'cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ' cc dd' - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', ' cc dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + ['cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', 'cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - r = f + [' cc', 'dd'] - assert isinstance(r, CLVar), type(r) - assert r.data == ['aa', 'bb', ' cc', 'dd'], r.data - assert str(r) == 'aa bb cc dd', str(r) - - # make sure inplace adding a string works as well (issue 2399) - # UserList would convert the string to a list of chars - f = CLVar(['aa', 'bb']) - f += 'cc dd' - assert isinstance(f, CLVar), type(f) - assert f.data == ['aa', 'bb', 'cc', 'dd'], f.data - assert str(f) == 'aa bb cc dd', str(f) - - f = CLVar(['aa', 'bb']) - f += ' cc dd' - assert isinstance(f, CLVar), type(f) - assert f.data == ['aa', 'bb', 'cc', 'dd'], f.data - assert str(f) == 'aa bb cc dd', str(f) - - - def test_Selector(self) -> None: - """Test the Selector class""" - - class MyNode: - def __init__(self, name) -> None: - self.name = name - - def __str__(self) -> str: - return self.name - - def get_suffix(self): - return os.path.splitext(self.name)[1] - - s = Selector({'a': 'AAA', 'b': 'BBB'}) - assert s['a'] == 'AAA', s['a'] - assert s['b'] == 'BBB', s['b'] - exc_caught = None - try: - x = s['c'] - except KeyError: - exc_caught = 1 - assert exc_caught, "should have caught a KeyError" - s['c'] = 'CCC' - assert s['c'] == 'CCC', s['c'] - - class DummyEnv(UserDict): - def subst(self, key): - if key[0] == '$': - return self[key[1:]] - return key - - env = DummyEnv() - - s = Selector({'.d': 'DDD', '.e': 'EEE'}) - ret = s(env, []) - assert ret is None, ret - ret = s(env, [MyNode('foo.d')]) - assert ret == 'DDD', ret - ret = s(env, [MyNode('bar.e')]) - assert ret == 'EEE', ret - ret = s(env, [MyNode('bar.x')]) - assert ret is None, ret - s[None] = 'XXX' - ret = s(env, [MyNode('bar.x')]) - assert ret == 'XXX', ret - - env = DummyEnv({'FSUFF': '.f', 'GSUFF': '.g'}) - - s = Selector({'$FSUFF': 'FFF', '$GSUFF': 'GGG'}) - ret = s(env, [MyNode('foo.f')]) - assert ret == 'FFF', ret - ret = s(env, [MyNode('bar.g')]) - assert ret == 'GGG', ret - - def test_adjustixes(self) -> None: - """Test the adjustixes() function""" - r = adjustixes('file', 'pre-', '-suf') - assert r == 'pre-file-suf', r - r = adjustixes('pre-file', 'pre-', '-suf') - assert r == 'pre-file-suf', r - r = adjustixes('file-suf', 'pre-', '-suf') - assert r == 'pre-file-suf', r - r = adjustixes('pre-file-suf', 'pre-', '-suf') - assert r == 'pre-file-suf', r - r = adjustixes('pre-file.xxx', 'pre-', '-suf') - assert r == 'pre-file.xxx', r - r = adjustixes('dir/file', 'pre-', '-suf') - assert r == os.path.join('dir', 'pre-file-suf'), r - - # Verify that the odd case when library name is specified as 'lib' - # doesn't yield lib.so, but yields the expected liblib.so - r = adjustixes('PREFIX', 'PREFIX', 'SUFFIX') - assert r == 'PREFIXPREFIXSUFFIX', "Failed handling when filename = PREFIX [r='%s']" % r - - def test_containsAny(self) -> None: - """Test the containsAny() function""" - assert containsAny('*.py', '*?[]') - assert not containsAny('file.txt', '*?[]') - - def test_containsAll(self) -> None: - """Test the containsAll() function""" - assert containsAll('43221', '123') - assert not containsAll('134', '123') - - def test_containsOnly(self) -> None: - """Test the containsOnly() function""" - assert containsOnly('.83', '0123456789.') - assert not containsOnly('43221', '123') - - def test_LogicalLines(self) -> None: - """Test the LogicalLines class""" - content = """ -foo \\ -bar \\ -baz -foo -bling \\ -bling \\ bling -bling -""" - fobj = io.StringIO(content) - lines = LogicalLines(fobj).readlines() - assert lines == [ - '\n', - 'foo bar baz\n', - 'foo\n', - 'bling bling \\ bling\n', - 'bling\n', - ], lines - - def test_intern(self) -> None: - s1 = silent_intern("spam") - s3 = silent_intern(42) - s4 = silent_intern("spam") - assert id(s1) == id(s4) - - -class HashTestCase(unittest.TestCase): - - def test_collect(self) -> None: - """Test collecting a list of signatures into a new signature value - """ - for algorithm, expected in { - 'md5': ('698d51a19d8a121ce581499d7b701668', - '8980c988edc2c78cc43ccb718c06efd5', - '53fd88c84ff8a285eb6e0a687e55b8c7'), - 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', - '42eda1b5dcb3586bccfb1c69f22f923145271d97', - '2eb2f7be4e883ebe52034281d818c91e1cf16256'), - 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', - '25235f0fcab8767b7b5ac6568786fbc4f7d5d83468f0626bf07c3dbeed391a7a', - 'f8d3d0729bf2427e2e81007588356332e7e8c4133fae4bceb173b93f33411d17'), - }.items(): - # if the current platform does not support the algorithm we're looking at, - # skip the test steps for that algorithm, but display a warning to the user - if algorithm not in ALLOWED_HASH_FORMATS: - warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) - else: - hs = functools.partial(hash_signature, hash_format=algorithm) - s = list(map(hs, ('111', '222', '333'))) - - assert expected[0] == hash_collect(s[0:1], hash_format=algorithm) - assert expected[1] == hash_collect(s[0:2], hash_format=algorithm) - assert expected[2] == hash_collect(s, hash_format=algorithm) - - def test_MD5signature(self) -> None: - """Test generating a signature""" - for algorithm, expected in { - 'md5': ('698d51a19d8a121ce581499d7b701668', - 'bcbe3365e6ac95ea2c0343a2395834dd'), - 'sha1': ('6216f8a75fd5bb3d5f22b6f9958cdede3fc086c2', - '1c6637a8f2e1f75e06ff9984894d6bd16a3a36a9'), - 'sha256': ('f6e0a1e2ac41945a9aa7ff8a8aaa0cebc12a3bcc981a929ad5cf810a090e11ae', - '9b871512327c09ce91dd649b3f96a63b7408ef267c8cc5710114e629730cb61f'), - }.items(): - # if the current platform does not support the algorithm we're looking at, - # skip the test steps for that algorithm, but display a warning to the user - if algorithm not in ALLOWED_HASH_FORMATS: - warnings.warn("Missing hash algorithm {} on this platform, cannot test with it".format(algorithm), ResourceWarning) - else: - s = hash_signature('111', hash_format=algorithm) - assert expected[0] == s, s - - s = hash_signature('222', hash_format=algorithm) - assert expected[1] == s, s - -# this uses mocking out, which is platform specific, however, the FIPS -# behavior this is testing is also platform-specific, and only would be -# visible in hosts running Linux with the fips_mode kernel flag along -# with using OpenSSL. - -class FIPSHashTestCase(unittest.TestCase): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - - ############################### - # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 - self.fake_md5=lambda usedforsecurity=True: (usedforsecurity, 'md5') - self.fake_sha1=lambda usedforsecurity=True: (usedforsecurity, 'sha1') - self.fake_sha256=lambda usedforsecurity=True: (usedforsecurity, 'sha256') - ############################### - - ############################### - # hashlib mocks - md5Available = unittest.mock.Mock(md5=self.fake_md5) - del md5Available.sha1 - del md5Available.sha256 - self.md5Available=md5Available - - md5Default = unittest.mock.Mock(md5=self.fake_md5, sha1=self.fake_sha1) - del md5Default.sha256 - self.md5Default=md5Default - - sha1Default = unittest.mock.Mock(sha1=self.fake_sha1, sha256=self.fake_sha256) - del sha1Default.md5 - self.sha1Default=sha1Default - - sha256Default = unittest.mock.Mock(sha256=self.fake_sha256, **{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError}) - self.sha256Default=sha256Default - - all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) - self.all_throw=all_throw - - no_algorithms = unittest.mock.Mock() - del no_algorithms.md5 - del no_algorithms.sha1 - del no_algorithms.sha256 - del no_algorithms.nonexist - self.no_algorithms=no_algorithms - - unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) - del unsupported_algorithm.md5 - del unsupported_algorithm.sha1 - del unsupported_algorithm.sha256 - del unsupported_algorithm.unsupported - self.unsupported_algorithm=unsupported_algorithm - ############################### - - ############################### - # system version mocks - VersionInfo = namedtuple('VersionInfo', 'major minor micro releaselevel serial') - v3_8 = VersionInfo(3, 8, 199, 'super-beta', 1337) - v3_9 = VersionInfo(3, 9, 0, 'alpha', 0) - v4_8 = VersionInfo(4, 8, 0, 'final', 0) - - self.sys_v3_8 = unittest.mock.Mock(version_info=v3_8) - self.sys_v3_9 = unittest.mock.Mock(version_info=v3_9) - self.sys_v4_8 = unittest.mock.Mock(version_info=v4_8) - ############################### - - def test_basic_failover_bad_hashlib_hash_init(self) -> None: - """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), - the hash init function returns None""" - assert _attempt_init_of_python_3_9_hash_object(None) is None - - def test_basic_failover_bad_hashlib_hash_get(self) -> None: - """Tests that if the hashing function is entirely missing from hashlib (hashlib returns None), - the hash get function returns None""" - assert _attempt_get_hash_function("nonexist", self.no_algorithms) is None - - def test_usedforsecurity_flag_behavior(self) -> None: - """Test usedforsecurity flag -> should be set to 'True' on older versions of python, and 'False' on Python >= 3.9""" - for version, expected in { - self.sys_v3_8: (True, 'md5'), - self.sys_v3_9: (False, 'md5'), - self.sys_v4_8: (False, 'md5'), - }.items(): - assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected - - def test_automatic_default_to_md5(self) -> None: - """Test automatic default to md5 even if sha1 available""" - for version, expected in { - self.sys_v3_8: (True, 'md5'), - self.sys_v3_9: (False, 'md5'), - self.sys_v4_8: (False, 'md5'), - }.items(): - _set_allowed_viable_default_hashes(self.md5Default, version) - set_hash_format(None, self.md5Default, version) - assert _get_hash_object(None, self.md5Default, version) == expected - - def test_automatic_default_to_sha256(self) -> None: - """Test automatic default to sha256 if other algorithms available but throw""" - for version, expected in { - self.sys_v3_8: (True, 'sha256'), - self.sys_v3_9: (False, 'sha256'), - self.sys_v4_8: (False, 'sha256'), - }.items(): - _set_allowed_viable_default_hashes(self.sha256Default, version) - set_hash_format(None, self.sha256Default, version) - assert _get_hash_object(None, self.sha256Default, version) == expected - - def test_automatic_default_to_sha1(self) -> None: - """Test automatic default to sha1 if md5 is missing from hashlib entirely""" - for version, expected in { - self.sys_v3_8: (True, 'sha1'), - self.sys_v3_9: (False, 'sha1'), - self.sys_v4_8: (False, 'sha1'), - }.items(): - _set_allowed_viable_default_hashes(self.sha1Default, version) - set_hash_format(None, self.sha1Default, version) - assert _get_hash_object(None, self.sha1Default, version) == expected - - def test_no_available_algorithms(self) -> None: - """expect exceptions on no available algorithms or when all algorithms throw""" - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.all_throw) - self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.unsupported_algorithm) - - def test_bad_algorithm_set_attempt(self) -> None: - """expect exceptions on user setting an unsupported algorithm selections, either by host or by SCons""" - - # nonexistant hash algorithm, not supported by SCons - _set_allowed_viable_default_hashes(self.md5Available) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) - - # md5 is default-allowed, but in this case throws when we attempt to use it - _set_allowed_viable_default_hashes(self.md5Available) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) - - # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons - _set_allowed_viable_default_hashes(self.sha1Default) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) - - # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons - _set_allowed_viable_default_hashes(self.sha1Default) - self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) - - def tearDown(self) -> None: - """Return SCons back to the normal global state for the hashing functions.""" - _set_allowed_viable_default_hashes(hashlib, sys) - set_hash_format(None) - - -class NodeListTestCase(unittest.TestCase): - def test_simple_attributes(self) -> None: - """Test simple attributes of a NodeList class""" - - class TestClass: - def __init__(self, name, child=None) -> None: - self.child = child - self.bar = name - - t1 = TestClass('t1', TestClass('t1child')) - t2 = TestClass('t2', TestClass('t2child')) - t3 = TestClass('t3') - - nl = NodeList([t1, t2, t3]) - assert nl.bar == ['t1', 't2', 't3'], nl.bar - assert nl[0:2].child.bar == ['t1child', 't2child'], \ - nl[0:2].child.bar - - def test_callable_attributes(self) -> None: - """Test callable attributes of a NodeList class""" - - class TestClass: - def __init__(self, name, child=None) -> None: - self.child = child - self.bar = name - - def foo(self): - return self.bar + "foo" - - def getself(self): - return self - - t1 = TestClass('t1', TestClass('t1child')) - t2 = TestClass('t2', TestClass('t2child')) - t3 = TestClass('t3') - - nl = NodeList([t1, t2, t3]) - assert nl.foo() == ['t1foo', 't2foo', 't3foo'], nl.foo() - assert nl.bar == ['t1', 't2', 't3'], nl.bar - assert nl.getself().bar == ['t1', 't2', 't3'], nl.getself().bar - assert nl[0:2].child.foo() == ['t1childfoo', 't2childfoo'], \ - nl[0:2].child.foo() - assert nl[0:2].child.bar == ['t1child', 't2child'], \ - nl[0:2].child.bar - - def test_null(self): - """Test a null NodeList""" - nl = NodeList([]) - r = str(nl) - assert r == '', r - for node in nl: - raise Exception("should not enter this loop") - - -class flattenTestCase(unittest.TestCase): - - def test_scalar(self) -> None: - """Test flattening a scalar""" - result = flatten('xyz') - self.assertEqual(result, ['xyz'], result) - - def test_dictionary_values(self) -> None: - """Test flattening the dictionary values""" - items = {"a": 1, "b": 2, "c": 3} - result = flatten(items.values()) - self.assertEqual(sorted(result), [1, 2, 3]) - - -class OsEnviron: - """Used to temporarily mock os.environ""" - - def __init__(self, environ) -> None: - self._environ = environ - - def start(self) -> None: - self._stored = os.environ - os.environ = self._environ - - def stop(self) -> None: - os.environ = self._stored - del self._stored - - def __enter__(self): - self.start() - return os.environ - - def __exit__(self, *args) -> None: - self.stop() - - -class get_env_boolTestCase(unittest.TestCase): - def test_missing(self) -> None: - env = dict() - var = get_env_bool(env, 'FOO') - assert var is False, "var should be False, not %s" % repr(var) - env = {'FOO': '1'} - var = get_env_bool(env, 'BAR') - assert var is False, "var should be False, not %s" % repr(var) - - def test_true(self) -> None: - for foo in ['TRUE', 'True', 'true', - 'YES', 'Yes', 'yes', - 'Y', 'y', - 'ON', 'On', 'on', - '1', '20', '-1']: - env = {'FOO': foo} - var = get_env_bool(env, 'FOO') - assert var is True, 'var should be True, not %s' % repr(var) - - def test_false(self) -> None: - for foo in ['FALSE', 'False', 'false', - 'NO', 'No', 'no', - 'N', 'n', - 'OFF', 'Off', 'off', - '0']: - env = {'FOO': foo} - var = get_env_bool(env, 'FOO', True) - assert var is False, 'var should be True, not %s' % repr(var) - - def test_default(self) -> None: - env = {'FOO': 'other'} - var = get_env_bool(env, 'FOO', True) - assert var is True, 'var should be True, not %s' % repr(var) - var = get_env_bool(env, 'FOO', False) - assert var is False, 'var should be False, not %s' % repr(var) - - -class get_os_env_boolTestCase(unittest.TestCase): - def test_missing(self) -> None: - with OsEnviron(dict()): - var = get_os_env_bool('FOO') - assert var is False, "var should be False, not %s" % repr(var) - with OsEnviron({'FOO': '1'}): - var = get_os_env_bool('BAR') - assert var is False, "var should be False, not %s" % repr(var) - - def test_true(self) -> None: - for foo in ['TRUE', 'True', 'true', - 'YES', 'Yes', 'yes', - 'Y', 'y', - 'ON', 'On', 'on', - '1', '20', '-1']: - with OsEnviron({'FOO': foo}): - var = get_os_env_bool('FOO') - assert var is True, 'var should be True, not %s' % repr(var) - - def test_false(self) -> None: - for foo in ['FALSE', 'False', 'false', - 'NO', 'No', 'no', - 'N', 'n', - 'OFF', 'Off', 'off', - '0']: - with OsEnviron({'FOO': foo}): - var = get_os_env_bool('FOO', True) - assert var is False, 'var should be True, not %s' % repr(var) - - def test_default(self) -> None: - with OsEnviron({'FOO': 'other'}): - var = get_os_env_bool('FOO', True) - assert var is True, 'var should be True, not %s' % repr(var) - var = get_os_env_bool('FOO', False) - assert var is False, 'var should be False, not %s' % repr(var) - - -if __name__ == "__main__": - unittest.main() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/doc/sphinx/SCons.Util.rst b/doc/sphinx/SCons.Util.rst new file mode 100644 index 0000000..553db7a --- /dev/null +++ b/doc/sphinx/SCons.Util.rst @@ -0,0 +1,36 @@ +SCons.Util package +================== + +Submodules +---------- + +.. automodule:: SCons.Util + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.envs + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.filelock + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.hashes + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.sctypes + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.stats + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/sphinx/SCons.rst b/doc/sphinx/SCons.rst index 45e20ee..85f5878 100644 --- a/doc/sphinx/SCons.rst +++ b/doc/sphinx/SCons.rst @@ -20,6 +20,7 @@ Subpackages SCons.Script SCons.Taskmaster SCons.Tool + SCons.Util SCons.Variables SCons.compat @@ -140,14 +141,6 @@ SCons.Subst module :undoc-members: :show-inheritance: -SCons.Util module ------------------ - -.. automodule:: SCons.Util - :members: - :undoc-members: - :show-inheritance: - SCons.Warnings module --------------------- diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index f8d5f47..04bfdc2 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -17,7 +17,7 @@ SCons API Documentation The target audience is developers working on SCons itself: what is "Public API" is not clearly deliniated here. The interfaces available for use in SCons configuration scripts, - which have a consistency guarantee, are those documented in the + which have a consistency guarantee, are those documented in the `SCons Reference Manual `_. @@ -33,6 +33,7 @@ SCons API Documentation SCons.Script SCons.Taskmaster SCons.Tool + SCons.Util SCons.Variables -- cgit v0.12 From 7e5911b7ff3d4d4595a21cada6681113e7c4a0fb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 22 Sep 2023 08:09:19 -0600 Subject: Make SCons.Util.types renaming explicit in notes [ci skip] Signed-off-by: Mats Wichmann --- CHANGES.txt | 5 ++++- RELEASE.txt | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0abd561..b4afba7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -168,7 +168,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER builds. Also add a simple filesystem-based locking protocol to try to avoid the problem occuring. - Update the first two chapters on building with SCons in the User Guide. - - Some cleanup to the Util package. + - Some cleanup to the Util package, including renaming SCons.Util.types + to SCons.Util.sctypes to avoid any possible confusion with the + Python stdlib types module. + From Jonathon Reinhart: - Fix another instance of `int main()` in CheckLib() causing failures diff --git a/RELEASE.txt b/RELEASE.txt index e45284c..e704614 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -72,6 +72,12 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY The "--warn=missing-sconscript" commandline option is no longer available as the warning was part of the transitional phase. - Add missing directories to searched paths for mingw installs +- SCons.Util.types renamed to to SCons.Util.sctypes to avoid any possible + confusion with the Python stdlib "types" module. Note that it was briefly + (for 4.5.x only) possible to import directly from SCons.Util.types, + although the preferred usage remains to import from SCons.Util only. + Any code that did the direct import will have to change to import from + SCons.Util.sctypes. FIXES ----- -- cgit v0.12