From 52773da0faddc808056ffec34ed0112e2b42b5f7 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 15 May 2020 23:31:41 -0400 Subject: use signature to check function signature instead of relying on TypeErrors --- SCons/Subst.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/SCons/Subst.py b/SCons/Subst.py index a1ef053..29698d5 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -30,7 +30,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import collections import re - +from inspect import signature import SCons.Errors from SCons.Util import is_String, is_Sequence @@ -420,12 +420,13 @@ class StringSubber(object): return conv(substitute(l, lvars)) return list(map(func, s)) elif callable(s): - try: + if (s and + set(signature(s).parameters.keys()) == set(['target', 'source', 'env', 'for_signature'])): s = s(target=lvars['TARGETS'], source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) - except TypeError: + else: # This probably indicates that it's a callable # object that doesn't match our calling arguments # (like an Action). -- cgit v0.12 From cffd40b44080dcce1d238d037e026051eaa7a60b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sun, 17 May 2020 15:41:12 -0400 Subject: cover other type of subber and add test --- SCons/Subst.py | 5 +++-- test/Subst/TypeError.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/SCons/Subst.py b/SCons/Subst.py index 29698d5..fbb89b3 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -591,12 +591,13 @@ class ListSubber(collections.UserList): self.substitute(a, lvars, 1) self.next_word() elif callable(s): - try: + if (s and + set(signature(s).parameters.keys()) == set(['target', 'source', 'env', 'for_signature'])): s = s(target=lvars['TARGETS'], source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) - except TypeError: + else: # This probably indicates that it's a callable # object that doesn't match our calling arguments # (like an Action). diff --git a/test/Subst/TypeError.py b/test/Subst/TypeError.py index 628db2f..b288961 100644 --- a/test/Subst/TypeError.py +++ b/test/Subst/TypeError.py @@ -85,8 +85,36 @@ expect = expect_build % (r' \[foo\.bar\]', r'\$\{func\(1\)\}') test.run(status=2, stderr=expect) +# callable exceptions: +test.write('foo.c', """\ +#include +#include +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c"); + exit (0); +} +""") + +test.write('SConstruct', """\ +class TestCallable(object): + def __init__(self, thing, makePathsRelative = True, debug = False): + pass + def __call__(self, target, source, env, for_signature): + raise TypeError("User callable exception") + +env = Environment() +env["TESTCLASS"] = TestCallable +env["CCCOM"] = "$CC $_CCCOMCOM $CCFLAGS -o ${TESTCLASS('$TARGET')} -c ${TESTCLASS('$SOURCES')}" + +env.Program(target='foo', source='foo.c') +""") +test.run(status=2, stderr=r'.*TypeError\s:\sUser\scallable\sexception.*') +print(test.stdout()) test.pass_test() # Local Variables: -- cgit v0.12 From 2a9eb7778a553d44bb2103953b4d563c9dab59d7 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sun, 17 May 2020 23:17:43 -0400 Subject: update CHANGES.txt --- CHANGES.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 431ecc1..e22dafd 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,12 +10,6 @@ NOTE: Please include a reference to any Issues resolved by your changes in the b RELEASE VERSION/DATE TO BE FILLED IN LATER - From Daniel Moody: - - Add no_progress (-Q) option as a set-able option. However, setting it in the - SConstruct/SConscript will still cause "scons: Reading SConscript files ..." to be - printed, since the option is not set when the build scripts first get read. - - Added check for SONAME in environment to setup symlinks correctly (Github Issue #3246) - From James Benton: - Improve Visual Studio solution/project generation code to add support for a per-variant cppflags. Intellisense can be affected by cppflags, @@ -86,6 +80,16 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER macro which is not located in a state TABLE. * Cleanup CPP expressions before evaluating (strip comments, carriage returns) + From Daniel Moody: + - Add no_progress (-Q) option as a set-able option. However, setting it in the + SConstruct/SConscript will still cause "scons: Reading SConscript files ..." to be + printed, since the option is not set when the build scripts first get read. + - Added check for SONAME in environment to setup symlinks correctly (Github Issue #3246) + - User callable's called during substition expansion could possibly throw a TypeError + exception, however SCons was using TypeError to detect if the callable had a different + signature than expected, and would silently fail to report user's exceptions. Fixed to + use signature module to detect function signature instead of TypeError. (Github Issue #3654) + From Andrew Morrow: - Fix Issue #3469 - Fixed improper reuse of temporary and compiled files by Configure when changing the order and/or number of tests. This is done by using the hash of the generated temporary files -- cgit v0.12 From 23a99693e3603a47f4bcb7b08351a774923b0f2e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 18 May 2020 17:24:40 -0400 Subject: add check for SCons null class --- SCons/Subst.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/SCons/Subst.py b/SCons/Subst.py index fbb89b3..cc2eeab 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -420,7 +420,10 @@ class StringSubber(object): return conv(substitute(l, lvars)) return list(map(func, s)) elif callable(s): - if (s and + # SCons has the unusual Null class where any __getattr__ call returns it's self, + # which does not work the signature module, and the Null class returns an empty + # string if called on, so we make an exception in this condition for Null class + if (isinstance(s, SCons.Util.Null) or set(signature(s).parameters.keys()) == set(['target', 'source', 'env', 'for_signature'])): s = s(target=lvars['TARGETS'], source=lvars['SOURCES'], @@ -591,7 +594,10 @@ class ListSubber(collections.UserList): self.substitute(a, lvars, 1) self.next_word() elif callable(s): - if (s and + # SCons has the unusual Null class where any __getattr__ call returns it's self, + # which does not work the signature module, and the Null class returns an empty + # string if called on, so we make an exception in this condition for Null class + if (isinstance(s, SCons.Util.Null) or set(signature(s).parameters.keys()) == set(['target', 'source', 'env', 'for_signature'])): s = s(target=lvars['TARGETS'], source=lvars['SOURCES'], -- cgit v0.12 From deacd0898818a0358fc29e0edff387acda5b7bc5 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 19 May 2020 17:14:41 -0400 Subject: used fixtures in test TypeError test --- test/Subst/TypeError.py | 32 +++--------------------- test/Subst/fixture/SConstruct.callable_exception | 11 ++++++++ 2 files changed, 15 insertions(+), 28 deletions(-) create mode 100644 test/Subst/fixture/SConstruct.callable_exception diff --git a/test/Subst/TypeError.py b/test/Subst/TypeError.py index b288961..c52434f 100644 --- a/test/Subst/TypeError.py +++ b/test/Subst/TypeError.py @@ -85,36 +85,12 @@ expect = expect_build % (r' \[foo\.bar\]', r'\$\{func\(1\)\}') test.run(status=2, stderr=expect) -# callable exceptions: -test.write('foo.c', """\ -#include -#include -int -main(int argc, char *argv[]) -{ - argv[argc++] = "--"; - printf("foo.c"); - exit (0); -} -""") - -test.write('SConstruct', """\ - -class TestCallable(object): - def __init__(self, thing, makePathsRelative = True, debug = False): - pass - def __call__(self, target, source, env, for_signature): - raise TypeError("User callable exception") - -env = Environment() -env["TESTCLASS"] = TestCallable -env["CCCOM"] = "$CC $_CCCOMCOM $CCFLAGS -o ${TESTCLASS('$TARGET')} -c ${TESTCLASS('$SOURCES')}" - -env.Program(target='foo', source='foo.c') -""") +# user callable exceptions (Github issue #3654): +test.file_fixture('test_main.c') +test.file_fixture('./fixture/SConstruct.callable_exception', 'SConstruct') test.run(status=2, stderr=r'.*TypeError\s:\sUser\scallable\sexception.*') -print(test.stdout()) + test.pass_test() # Local Variables: diff --git a/test/Subst/fixture/SConstruct.callable_exception b/test/Subst/fixture/SConstruct.callable_exception new file mode 100644 index 0000000..3f1b337 --- /dev/null +++ b/test/Subst/fixture/SConstruct.callable_exception @@ -0,0 +1,11 @@ +class TestCallable(object): + def __init__(self, thing, makePathsRelative = True, debug = False): + pass + def __call__(self, target, source, env, for_signature): + raise TypeError("User callable exception") + +env = Environment() +env["TESTCLASS"] = TestCallable +env["CCCOM"] = "$CC $_CCCOMCOM $CCFLAGS -o ${TESTCLASS('$TARGET')} -c ${TESTCLASS('$SOURCES')}" + +env.Program(target='test_main', source='test_main.c') \ No newline at end of file -- cgit v0.12 From 8662a4efdf65dd5be31315c23338dc13d898498b Mon Sep 17 00:00:00 2001 From: "Andrii Doroshenko (Xrayez)" Date: Wed, 27 May 2020 19:43:13 +0300 Subject: Extend `Environment.Dump()` to select serialization format Environment.Dump() produces pretty-printable results only, so the usefulness of this method is limited to debugging purposes. The existing method is extended to allow selecting a serialization format to use via a `format` parameter. Namely, it's now possible to serialize variables as a JSON-formatted string, which makes it possible for automated external tools to inspect the environment more easily. --- CHANGES.txt | 3 +++ SCons/Environment.py | 36 ++++++++++++++++++++++++++---------- SCons/Environment.xml | 27 +++++++++++++++++++++++++-- SCons/EnvironmentTests.py | 12 ++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ecf3255..4774881 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -54,6 +54,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Reorganized source tree. Moved src/engine/SCons to SCons to be more in line with current Python source tree organization practices. + From Andrii Doroshenko: + - Extended `Environment.Dump()` to select a format to serialize construction variables (pretty, json). + From Jeremy Elson: - Updated design doc to use the correct syntax for Depends() diff --git a/SCons/Environment.py b/SCons/Environment.py index 0b4be1b..cd52ee5 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -1517,26 +1517,42 @@ class Base(SubstitutionEnvironment): return dlist - def Dump(self, key=None): - """ Return pretty-printed string of construction variables. + def Dump(self, key=None, format='pretty'): + """ Serialize the construction variables to a string. :param key: if None, format the whole dict of variables. Else look up and format just the value for key. + + :param format: specify the format of the variables to be serialized: + - pretty: pretty-printed string. + - json: JSON-formatted string. """ - import pprint - pp = pprint.PrettyPrinter(indent=2) if key: cvars = self.Dictionary(key) else: cvars = self.Dictionary() - # TODO: pprint doesn't do a nice job on path-style values - # if the paths contain spaces (i.e. Windows), because the - # algorithm tries to break lines on spaces, while breaking - # on the path-separator would be more "natural". Is there - # a better way to format those? - return pp.pformat(cvars) + fmt = format.lower() + + if fmt == 'pretty': + import pprint + pp = pprint.PrettyPrinter(indent=2) + + # TODO: pprint doesn't do a nice job on path-style values + # if the paths contain spaces (i.e. Windows), because the + # algorithm tries to break lines on spaces, while breaking + # on the path-separator would be more "natural". Is there + # a better way to format those? + return pp.pformat(cvars) + + elif fmt == 'json': + import json + def non_serializable(obj): + return str(type(obj).__qualname__) + return json.dumps(cvars, indent=4, default=non_serializable) + else: + raise ValueError("Unsupported serialization format: %s." % fmt) def FindIxes(self, paths, prefix, suffix): diff --git a/SCons/Environment.xml b/SCons/Environment.xml index d13d560..898dc5d 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -1393,11 +1393,34 @@ for more information. -([key]) +([key], [format]) -Returns a pretty printable representation of the environment. +Serializes the construction variables to a string. +The method supports the following formats specified by +format: + + +pretty + + +Returns a pretty printable representation of the environment (if +format +is not specified, this is the default). + + + + +json + + +Returns a JSON-formatted string representation of the environment. + + + + + key, if not None, diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index ef12a6f..6dcf3c1 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -2941,6 +2941,18 @@ def generate(env): assert env.Dump('FOO') == "'foo'", env.Dump('FOO') assert len(env.Dump()) > 200, env.Dump() # no args version + assert env.Dump('FOO', 'json') == '"foo"' # JSON key version + import json + env_dict = json.loads(env.Dump(format = 'json')) + assert env_dict['FOO'] == 'foo' # full JSON version + + try: + env.Dump(format = 'markdown') + except ValueError as e: + assert str(e) == "Unsupported serialization format: markdown." + else: + self.fail("Did not catch expected ValueError.") + def test_Environment(self): """Test the Environment() method""" env = self.TestEnvironment(FOO = 'xxx', BAR = 'yyy') -- cgit v0.12 From 46810dae9a306c934f89ad6b3f37cc82eb2026a5 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 29 May 2020 11:14:49 -0600 Subject: Close scons logfiles on completion Files written to in logging operations could remain unclosed: more modern Pythons grumble about this; given the type of Python build, could emit ResourceWarning messages which cause tests to fail. Close by registering calls with atexit. Affects Trace, cache debug, taskmastertrace, configure. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ SCons/CacheDir.py | 7 ++++++- SCons/Debug.py | 34 ++++++++++++++++++++++++++-------- SCons/SConf.py | 10 ++++++++-- SCons/Script/Main.py | 5 +++++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ecf3255..88b8d6f 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -150,6 +150,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - ParseFlags should not modify the user's passed in dict in case it's a compound data structure (e.g. values are lists) (issue #3665) - In Py3 classes no longer need to be listed as deriving from object. + - Close various logfiles (trace, cache, taskmastertrace, configure) + when done using atexit calls. diff --git a/SCons/CacheDir.py b/SCons/CacheDir.py index 5ecbc6f..3bb6671 100644 --- a/SCons/CacheDir.py +++ b/SCons/CacheDir.py @@ -27,13 +27,14 @@ __doc__ = """ CacheDir support """ +import atexit import json import os import stat import sys -import SCons import SCons.Action +import SCons.Errors import SCons.Warnings cache_enabled = True @@ -202,7 +203,11 @@ class CacheDir: if cache_debug == '-': self.debugFP = sys.stdout elif cache_debug: + def debug_cleanup(debugFP): + debugFP.close() + self.debugFP = open(cache_debug, 'w') + atexit.register(debug_cleanup, self.debugFP) else: self.debugFP = None self.current_cache_debug = cache_debug diff --git a/SCons/Debug.py b/SCons/Debug.py index 706b4c4..10cd2f5 100644 --- a/SCons/Debug.py +++ b/SCons/Debug.py @@ -33,6 +33,7 @@ caller_trace() __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import atexit import os import sys import time @@ -66,7 +67,7 @@ def string_to_classes(s): def fetchLoggedInstances(classes="*"): classnames = string_to_classes(classes) return [(cn, len(tracked_classes[cn])) for cn in classnames] - + def countLoggedInstances(classes, file=sys.stdout): for classname in string_to_classes(classes): file.write("%s: %d\n" % (classname, len(tracked_classes[classname]))) @@ -201,22 +202,39 @@ if sys.platform == 'win32': TraceDefault = 'con' else: TraceDefault = '/dev/tty' - -TimeStampDefault = None +TimeStampDefault = False StartTime = time.time() PreviousTime = StartTime -def Trace(msg, file=None, mode='w', tstamp=None): - """Write a trace message to a file. Whenever a file is specified, - it becomes the default for the next call to Trace().""" +def Trace(msg, filename=None, mode='w', tstamp=False): + """Write a trace message. + + Write messages when debugging which do not interfere with stdout. + Useful in tests, which monitor stdout and would break with + unexpected output. Trace messages can go to the console (which is + opened as a file), or to a disk file; the file argument persists + across calls unless overridden. + + Args: + filename: file to write trace message to. If omitted, + write to the previous trace file (default: console). + mode: file open mode (default: 'w') + tstamp: write relative timestamps with trace. Outputs time since + scons was started, and time since last trace (default: False) + + """ global TraceDefault global TimeStampDefault global PreviousTime + + def traace_cleanup(traceFP): + traceFP.close() + if file is None: file = TraceDefault else: TraceDefault = file - if tstamp is None: + if not tstamp: tstamp = TimeStampDefault else: TimeStampDefault = tstamp @@ -225,6 +243,7 @@ def Trace(msg, file=None, mode='w', tstamp=None): except KeyError: try: fp = TraceFP[file] = open(file, mode) + atexit.register(trace_cleanup, fp) except TypeError: # Assume we were passed an open file pointer. fp = file @@ -234,7 +253,6 @@ def Trace(msg, file=None, mode='w', tstamp=None): PreviousTime = now fp.write(msg) fp.flush() - fp.close() # Local Variables: # tab-width:4 diff --git a/SCons/SConf.py b/SCons/SConf.py index 42bfa8c..efa9e5b 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -37,6 +37,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.compat +import atexit import io import os import re @@ -750,11 +751,16 @@ class SConfBase: _ac_config_logs[self.logfile] = None log_mode = "w" fp = open(str(self.logfile), log_mode) + + def conflog_cleanup(logf): + logf.close() + + atexit.register(conflog_cleanup, fp) self.logstream = SCons.Util.Unbuffered(fp) # logfile may stay in a build directory, so we tell - # the build system not to override it with a eventually + # the build system not to override it with an eventually # existing file with the same name in the source directory - self.logfile.dir.add_ignore( [self.logfile] ) + self.logfile.dir.add_ignore([self.logfile]) tb = traceback.extract_stack()[-3-self.depth] old_fs_dir = SConfFS.getcwd() diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index ac29712..d7b07a2 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -40,6 +40,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.compat +import atexit import importlib.util import os import re @@ -1262,10 +1263,14 @@ def _build_targets(fs, options, targets, target_top): """Leave the order of dependencies alone.""" return dependencies + def tmtrace_cleanup(tfile): + tfile.close() + if options.taskmastertrace_file == '-': tmtrace = sys.stdout elif options.taskmastertrace_file: tmtrace = open(options.taskmastertrace_file, 'w') + atexit.register(tmtrace_cleanup, tmtrace) else: tmtrace = None taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) -- cgit v0.12 From 5708798e8443376cb52819dc1bcae3a59ed94ea6 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 29 May 2020 12:25:45 -0600 Subject: Fix typo in new trace_cleanup interior func Spotted by sider Signed-off-by: Mats Wichmann --- SCons/Debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Debug.py b/SCons/Debug.py index 10cd2f5..2212eb8 100644 --- a/SCons/Debug.py +++ b/SCons/Debug.py @@ -227,7 +227,7 @@ def Trace(msg, filename=None, mode='w', tstamp=False): global TimeStampDefault global PreviousTime - def traace_cleanup(traceFP): + def trace_cleanup(traceFP): traceFP.close() if file is None: -- cgit v0.12