From 9eb21f94e0490e8e7811f1033ea146acac0b7c34 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 16 Feb 2019 11:10:29 -0600 Subject: update lex tool to find paths on windows --- src/CHANGES.txt | 5 +++-- src/engine/SCons/Tool/lex.py | 44 +++++++++++++++++++++++++++++++++++++++++--- test/LEX/live.py | 18 +++++++++++++----- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 0d02ec0..bfea661 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -19,6 +19,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Change the default for AppendENVPath to delete_existing=0, so path order will not be changed, unless explicitly set (Issue #3276) - Fixed bug which threw error when running SCons on windows system with no MSVC installed. + - Update link tool to convert target to node before accessing node member + - Update mingw tool to remove MSVC like nologo CCFLAG + - Update lex tool to work in mingw environments From Mats Wichmann: - Quiet open file ResourceWarnings on Python >= 3.6 caused by @@ -42,8 +45,6 @@ RELEASE 3.0.4 - Mon, 20 Jan 2019 22:49:27 +0000 Issues #3268 & Issue #3222 - Initial support for ARM targets with Visual Studio 2017 - Issue #3182 (You must set TARGET_ARCH for this to work) - Update TempFileMunge class to use PRINT_CMD_LINE_FUNC - - Update link tool to convert target to node before accessing node member - - Update mingw tool to remove MSVC like nologo CCFLAG From Tobias Herzog - Enhance cpp scanner regex logic to detect if/elif expressions without whitespaces but diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index 280c768..732c0da 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -34,10 +34,13 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path +import sys import SCons.Action import SCons.Tool import SCons.Util +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR") @@ -64,10 +67,42 @@ def lexEmitter(target, source, env): target.append(fileName) return (target, source) +def get_lex_path(env, append_paths=False): + """ + Find the a path containing the lex or flex binaries. If a construction + environment is passed in then append the path to the ENV PATH. + """ + # save existing path to reset if we don't want to append any paths + envPath = env['ENV']['PATH'] + + lex = SCons.Tool.find_program_path(env, 'lex', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + if lex: + if not append_paths: + env['ENV']['PATH'] = envPath + else: + lex_bin_dir = os.path.dirname(lex) + env.AppendENVPath('PATH', lex_bin_dir) + return lex + + flex = SCons.Tool.find_program_path(env, 'flex', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + if flex: + if not append_paths: + env['ENV']['PATH'] = envPath + else: + flex_bin_dir = os.path.dirname(flex) + env.AppendENVPath('PATH', flex_bin_dir) + return flex + else: + SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') + + def generate(env): """Add Builders and construction variables for lex to an Environment.""" c_file, cxx_file = SCons.Tool.createCFileBuilders(env) + if sys.platform == 'win32': + get_lex_path(env, append_paths=True) + # C c_file.add_action(".l", LexAction) c_file.add_emitter(".l", lexEmitter) @@ -82,13 +117,16 @@ def generate(env): # C++ cxx_file.add_action(".ll", LexAction) cxx_file.add_emitter(".ll", lexEmitter) - - env["LEX"] = env.Detect("flex") or "lex" + + env["LEX"] = env.Detect("flex") or "lex" env["LEXFLAGS"] = SCons.Util.CLVar("") env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" def exists(env): - return env.Detect(["flex", "lex"]) + if sys.platform == 'win32': + return get_lex_path(env) + else: + return env.Detect(["flex", "lex"]) # Local Variables: # tab-width:4 diff --git a/test/LEX/live.py b/test/LEX/live.py index 3d697d5..853e97c 100644 --- a/test/LEX/live.py +++ b/test/LEX/live.py @@ -28,6 +28,8 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" Test LEX and LEXFLAGS with a live lex. """ +import sys + import TestSCons _exe = TestSCons._exe @@ -40,15 +42,24 @@ lex = test.where_is('lex') or test.where_is('flex') if not lex: test.skip_test('No lex or flex found; skipping test.\n') +tools = "'default'" +if sys.platform == 'win32': + # make sure mingw is installed on win32 + if not test.where_is('gcc'): + test.skip_test('No mingw on windows; skipping test.\n') + # lex on win32 has a dependencies on mingw for unix headers + # so add it as a tool to the environment. + tools += ", 'mingw'" test.file_fixture('wrapper.py') test.write('SConstruct', """ -foo = Environment() +foo = Environment(tools=[%(tools)s]) lex = foo.Dictionary('LEX') bar = Environment(LEX = r'%(_python_)s wrapper.py ' + lex, - LEXFLAGS = '-b') + LEXFLAGS = '-b', + tools=[%(tools)s]) foo.Program(target = 'foo', source = 'foo.l') bar.Program(target = 'bar', source = 'bar.l') """ % locals()) @@ -82,9 +93,6 @@ test.must_not_exist(test.workpath('lex.backup')) test.run(program = test.workpath('foo'), stdin = "a\n", stdout = "Afoo.lA\n") - - - test.run(arguments = 'bar' + _exe) test.must_match(test.workpath('wrapper.out'), "wrapper.py\n") -- cgit v0.12 From 2feb2f39ccd71cfd72f42c5f97eedbe23b5a5853 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 16 Feb 2019 11:10:46 -0600 Subject: add check for tar before trying to run test --- test/packaging/multiple-packages-at-once.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/packaging/multiple-packages-at-once.py b/test/packaging/multiple-packages-at-once.py index 8c3f72d..3575996 100644 --- a/test/packaging/multiple-packages-at-once.py +++ b/test/packaging/multiple-packages-at-once.py @@ -38,10 +38,14 @@ python = TestSCons.python test = TestSCons.TestSCons() zip = test.detect('ZIP', 'zip') +tar = test.detect('TAR', 'tar') if not zip: test.skip_test('zip not found, skipping test\n') +if not tar: + test.skip_test('tar not found, skipping test\n') + test.subdir('src') test.write( [ 'src', 'main.c' ], r""" -- cgit v0.12 From 77560936fe38dccf1c4c46a2b41e3c5d4e3a0462 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 16 Feb 2019 22:24:00 -0600 Subject: condensed and organized code --- src/engine/SCons/Tool/lex.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index 732c0da..70c0c5f 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -74,26 +74,18 @@ def get_lex_path(env, append_paths=False): """ # save existing path to reset if we don't want to append any paths envPath = env['ENV']['PATH'] - - lex = SCons.Tool.find_program_path(env, 'lex', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) - if lex: - if not append_paths: - env['ENV']['PATH'] = envPath - else: - lex_bin_dir = os.path.dirname(lex) - env.AppendENVPath('PATH', lex_bin_dir) - return lex - - flex = SCons.Tool.find_program_path(env, 'flex', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) - if flex: - if not append_paths: - env['ENV']['PATH'] = envPath - else: - flex_bin_dir = os.path.dirname(flex) - env.AppendENVPath('PATH', flex_bin_dir) - return flex - else: - SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') + bins = ['lex', 'flex'] + + for prog in bins: + bin_path = SCons.Tool.find_program_path(env, prog, default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + if bin_path: + if not append_paths: + env['ENV']['PATH'] = envPath + else: + env.AppendENVPath('PATH', os.path.dirname(bin_path)) + return bin_path + + SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') def generate(env): -- cgit v0.12 From 16d8a62b17ae8a3e026d192574a05bdb87140c59 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 20 Feb 2019 00:16:11 -0600 Subject: add win_flex as option for windows, add choco default path, and add flag for nounistd on windows. also more testing --- .appveyor.yml | 4 +- src/engine/SCons/Platform/win32.py | 4 ++ src/engine/SCons/Tool/lex.py | 26 +++++---- src/engine/SCons/Tool/lex.xml | 9 ++++ test/CFILESUFFIX.py | 6 ++- test/CXX/CXXFILESUFFIX.py | 6 ++- test/LEX/LEX.py | 6 ++- test/LEX/LEXFLAGS.py | 25 +++++---- test/LEX/live.py | 21 ++------ test/LEX/live_mingw.py | 107 +++++++++++++++++++++++++++++++++++++ test/LEX/no_lex.py | 59 ++++++++++++++++++++ 11 files changed, 231 insertions(+), 42 deletions(-) create mode 100644 test/LEX/live_mingw.py create mode 100644 test/LEX/no_lex.py diff --git a/.appveyor.yml b/.appveyor.yml index 1849707..cbf7233 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,12 +18,12 @@ install: - cmd: "C:\\%WINPYTHON%\\python.exe --version" - cmd: for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -m site --user-site') do (set PYSITEDIR=%%g) # use mingw 32 bit until #3291 is resolved - - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\ProgramData\\chocolatey\\bin;%PATH%" + - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;%PATH%" - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel " - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pypiwin32 coverage codecov" - cmd: set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml # install 3rd party tools to test with - - cmd: choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc + - cmd: choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc winflexbison - cmd: set ### LINUX ### diff --git a/src/engine/SCons/Platform/win32.py b/src/engine/SCons/Platform/win32.py index 2d40fb8..be30546 100644 --- a/src/engine/SCons/Platform/win32.py +++ b/src/engine/SCons/Platform/win32.py @@ -43,6 +43,10 @@ from SCons.Platform.virtualenv import ImportVirtualenv from SCons.Platform.virtualenv import ignore_virtualenv, enable_virtualenv import SCons.Util +CHOCO_DEFAULT_PATH = [ + r'C:\ProgramData\chocolatey\bin' +] + try: import msvcrt import win32api diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index 70c0c5f..3506f7c 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -41,6 +41,7 @@ import SCons.Tool import SCons.Util from SCons.Platform.mingw import MINGW_DEFAULT_PATHS from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS +from SCons.Platform.win32 import CHOCO_DEFAULT_PATH LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR") @@ -74,17 +75,19 @@ def get_lex_path(env, append_paths=False): """ # save existing path to reset if we don't want to append any paths envPath = env['ENV']['PATH'] - bins = ['lex', 'flex'] + bins = ['win_flex', 'lex', 'flex'] for prog in bins: - bin_path = SCons.Tool.find_program_path(env, prog, default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + bin_path = SCons.Tool.find_program_path( + env, + prog, + default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) if bin_path: if not append_paths: env['ENV']['PATH'] = envPath else: env.AppendENVPath('PATH', os.path.dirname(bin_path)) return bin_path - SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') @@ -92,9 +95,6 @@ def generate(env): """Add Builders and construction variables for lex to an Environment.""" c_file, cxx_file = SCons.Tool.createCFileBuilders(env) - if sys.platform == 'win32': - get_lex_path(env, append_paths=True) - # C c_file.add_action(".l", LexAction) c_file.add_emitter(".l", lexEmitter) @@ -109,10 +109,16 @@ def generate(env): # C++ cxx_file.add_action(".ll", LexAction) cxx_file.add_emitter(".ll", lexEmitter) - - env["LEX"] = env.Detect("flex") or "lex" - env["LEXFLAGS"] = SCons.Util.CLVar("") - env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" + + if sys.platform == 'win32': + get_lex_path(env, append_paths=True) + env["LEX"] = env.Detect(['win_flex', 'lex', 'flex']) + env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd") + env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET" + else: + env["LEX"] = env.Detect(["flex", "lex"]) + env["LEXFLAGS"] = SCons.Util.CLVar("") + env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" def exists(env): if sys.platform == 'win32': diff --git a/src/engine/SCons/Tool/lex.xml b/src/engine/SCons/Tool/lex.xml index 0388ee3..f933451 100644 --- a/src/engine/SCons/Tool/lex.xml +++ b/src/engine/SCons/Tool/lex.xml @@ -33,6 +33,7 @@ Sets construction variables for the &lex; lexical analyser. LEX LEXFLAGS LEXCOM +LEXUNISTD LEXCOMSTR @@ -78,4 +79,12 @@ General options passed to the lexical analyzer generator. + + + +Used only on windows environments to set a lex flag to prevent 'unistd.h' from being included. The default value is '--nounistd'. + + + + diff --git a/test/CFILESUFFIX.py b/test/CFILESUFFIX.py index 0a3a81a..410ece5 100644 --- a/test/CFILESUFFIX.py +++ b/test/CFILESUFFIX.py @@ -39,7 +39,11 @@ test = TestSCons.TestSCons() test.write('mylex.py', """ import getopt import sys -cmd_opts, args = getopt.getopt(sys.argv[1:], 't', []) +if sys.platform == 'win32': + longopts = ['nounistd'] +else: + longopts = [] +cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts) for a in args: contents = open(a, 'rb').read() sys.stdout.write((contents.replace(b'LEX', b'mylex.py')).decode()) diff --git a/test/CXX/CXXFILESUFFIX.py b/test/CXX/CXXFILESUFFIX.py index 9442408..c8dbf0a 100644 --- a/test/CXX/CXXFILESUFFIX.py +++ b/test/CXX/CXXFILESUFFIX.py @@ -35,7 +35,11 @@ test = TestSCons.TestSCons() test.write('mylex.py', """ import getopt import sys -cmd_opts, args = getopt.getopt(sys.argv[1:], 't', []) +if sys.platform == 'win32': + longopts = ['nounistd'] +else: + longopts = [] +cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts) for a in args: contents = open(a, 'r').read() sys.stdout.write(contents.replace('LEX', 'mylex.py')) diff --git a/test/LEX/LEX.py b/test/LEX/LEX.py index 1239c6b..65e4497 100644 --- a/test/LEX/LEX.py +++ b/test/LEX/LEX.py @@ -38,7 +38,11 @@ test = TestSCons.TestSCons() test.write('mylex.py', """ import getopt import sys -cmd_opts, args = getopt.getopt(sys.argv[1:], 't', []) +if sys.platform == 'win32': + longopts = ['nounistd'] +else: + longopts = [] +cmd_opts, args = getopt.getopt(sys.argv[1:], 't', longopts) for a in args: contents = open(a, 'rb').read() sys.stdout.write(contents.replace(b'LEX', b'mylex.py').decode()) diff --git a/test/LEX/LEXFLAGS.py b/test/LEX/LEXFLAGS.py index 54df161..51b6614 100644 --- a/test/LEX/LEXFLAGS.py +++ b/test/LEX/LEXFLAGS.py @@ -25,6 +25,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons @@ -35,22 +36,25 @@ test = TestSCons.TestSCons() test.subdir('in') - - test.write('mylex.py', """ import getopt import sys -cmd_opts, args = getopt.getopt(sys.argv[1:], 'I:tx', []) +import os +if sys.platform == 'win32': + longopts = ['nounistd'] +else: + longopts = [] +cmd_opts, args = getopt.getopt(sys.argv[1:], 'I:tx', longopts) opt_string = '' i_arguments = '' for opt, arg in cmd_opts: if opt == '-I': i_arguments = i_arguments + ' ' + arg else: opt_string = opt_string + ' ' + opt for a in args: - contents = open(a, 'rb').read() - contents = contents.replace(b'LEXFLAGS', opt_string.encode()) - contents = contents.replace(b'I_ARGS', i_arguments.encode()) - sys.stdout.write(contents.decode()) + contents = open(a, 'r').read() + contents = contents.replace('LEXFLAGS', opt_string) + contents = contents.replace('I_ARGS', i_arguments) + sys.stdout.write(contents) sys.exit(0) """) @@ -61,13 +65,16 @@ env = Environment(LEX = r'%(_python_)s mylex.py', env.CFile(target = 'out/aaa', source = 'in/aaa.l') """ % locals()) -test.write(['in', 'aaa.l'], "aaa.l\nLEXFLAGS\nI_ARGS\n") +test.write(['in', 'aaa.l'], "aaa.l\nLEXFLAGS\nI_ARGS\n") test.run('.', stderr = None) +lexflags = ' -x -t' +if sys.platform == 'win32': + lexflags = ' --nounistd' + lexflags # Read in with mode='r' because mylex.py implicitley wrote to stdout # with mode='w'. -test.must_match(['out', 'aaa.c'], "aaa.l\n -x -t\n out in\n", mode='r') +test.must_match(['out', 'aaa.c'], "aaa.l\n%s\n out in\n" % lexflags, mode='r') diff --git a/test/LEX/live.py b/test/LEX/live.py index 853e97c..91a2d49 100644 --- a/test/LEX/live.py +++ b/test/LEX/live.py @@ -28,8 +28,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" Test LEX and LEXFLAGS with a live lex. """ -import sys - import TestSCons _exe = TestSCons._exe @@ -37,29 +35,18 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() -lex = test.where_is('lex') or test.where_is('flex') +lex = test.where_is('win_flex') or test.where_is('lex') or test.where_is('flex') if not lex: test.skip_test('No lex or flex found; skipping test.\n') -tools = "'default'" -if sys.platform == 'win32': - # make sure mingw is installed on win32 - if not test.where_is('gcc'): - test.skip_test('No mingw on windows; skipping test.\n') - # lex on win32 has a dependencies on mingw for unix headers - # so add it as a tool to the environment. - tools += ", 'mingw'" - - test.file_fixture('wrapper.py') test.write('SConstruct', """ -foo = Environment(tools=[%(tools)s]) +foo = Environment() lex = foo.Dictionary('LEX') bar = Environment(LEX = r'%(_python_)s wrapper.py ' + lex, - LEXFLAGS = '-b', - tools=[%(tools)s]) + LEXFLAGS = '-b') foo.Program(target = 'foo', source = 'foo.l') bar.Program(target = 'bar', source = 'bar.l') """ % locals()) @@ -100,8 +87,6 @@ test.must_exist(test.workpath('lex.backup')) test.run(program = test.workpath('bar'), stdin = "b\n", stdout = "Bbar.lB\n") - - test.pass_test() # Local Variables: diff --git a/test/LEX/live_mingw.py b/test/LEX/live_mingw.py new file mode 100644 index 0000000..13e2342 --- /dev/null +++ b/test/LEX/live_mingw.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test LEX and LEXFLAGS and unistd.h with a live lex in mingw environment. +""" + +import sys + +import TestSCons + +_exe = TestSCons._exe +_python_ = TestSCons._python_ + +test = TestSCons.TestSCons() + +if sys.platform != 'win32': + test.skip_test('Not windows environment; skipping test.\n') + +if not test.where_is('gcc'): + test.skip_test('No mingw or cygwin on windows; skipping test.\n') + +lex = test.where_is('lex') or test.where_is('flex') + +if not lex: + test.skip_test('No lex or flex found; skipping test.\n') + +test.file_fixture('wrapper.py') + +test.write('SConstruct', """ +foo = Environment(tools=['default', 'mingw', 'lex'], LEXUNISTD="") +lex = foo.Dictionary('LEX') +bar = Environment(LEX = r'%(_python_)s wrapper.py ' + lex, + LEXFLAGS = '-b', + LEXUNISTD="", + tools=['default', 'mingw', 'lex']) +foo.Program(target = 'foo', source = 'foo.l') +bar.Program(target = 'bar', source = 'bar.l') +""" % locals()) + +lex = r""" +%%%% +a printf("A%sA"); +b printf("B%sB"); +%%%% +int +yywrap() +{ + return 1; +} + +int +main() +{ + yylex(); +} +""" + +test.write('foo.l', lex % ('foo.l', 'foo.l')) + +test.write('bar.l', lex % ('bar.l', 'bar.l')) + +test.run(arguments = 'foo' + _exe, stderr = None) + +test.must_not_exist(test.workpath('wrapper.out')) +test.must_not_exist(test.workpath('lex.backup')) + +test.run(program = test.workpath('foo'), stdin = "a\n", stdout = "Afoo.lA\n") + +test.run(arguments = 'bar' + _exe) + +test.must_match(test.workpath('wrapper.out'), "wrapper.py\n") +test.must_exist(test.workpath('lex.backup')) + +test.run(program = test.workpath('bar'), stdin = "b\n", stdout = "Bbar.lB\n") +test.must_contain(test.workpath('bar.c'), "unistd.h") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/LEX/no_lex.py b/test/LEX/no_lex.py new file mode 100644 index 0000000..89ffdc7 --- /dev/null +++ b/test/LEX/no_lex.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test Environments are functional and return None when no lex tool is found. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +import SCons + +def no_lex(env, key_program, default_paths=[]): + return None + +class TestEnvironment(SCons.Environment.Environment): + def Detect(self, progs): + return None + +SCons.Tool.find_program_path = no_lex + +foo = TestEnvironment(tools=['default', 'lex']) +print(foo.Dictionary('LEX')) +""" % locals()) + +test.run(arguments = '-Q -s', stdout = 'None\n' ) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From e863e1bc71aea05c554e6d06014b51c19f5a6bf5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Feb 2019 14:41:06 -0800 Subject: Add test for GH Issue #3303 --- test/Configure/option--config.py | 10 ++++++++++ test/fixture/test_main.c | 4 ++++ 2 files changed, 14 insertions(+) create mode 100644 test/fixture/test_main.c diff --git a/test/Configure/option--config.py b/test/Configure/option--config.py index 80a8bcd..02f10ce 100644 --- a/test/Configure/option--config.py +++ b/test/Configure/option--config.py @@ -122,6 +122,9 @@ test.checkLogAndStdout(["Checking for C header file non_system_header0.h... ", [((".c", CR), (_obj, NCR))]], "config.log", ".sconf_temp", "SConstruct") + +test.file_fixture('test_main.c') + # Check the combination of --config=force and Decider('MD5-timestamp') # On second run there was an issue where the decider would throw DeciderNeedsNode # exception which the configure code didn't handle. @@ -132,12 +135,19 @@ env.Decider('MD5-timestamp') conf = Configure(env) conf.TryLink('int main(){return 0;}','.c') env = conf.Finish() +env.Program('test_main.c') """) test.run(arguments='--config=force') # On second run the sconsign is loaded and decider doesn't just indicate need to rebuild test.run(arguments='--config=force') test.must_not_contain(test.workpath('config.log'), "TypeError: 'NoneType' object is not callable", mode='r') +# Now check to check that test_main.c didn't rebuild on second run above. +# This fixes an issue where --config=force overwrites the Environments decider and is not reset when +# the configure context is done. +# https://github.com/SCons/scons/issues/3303 +test.fail_test(test.stdout().find('test_main.o') != -1) + test.pass_test() # Local Variables: diff --git a/test/fixture/test_main.c b/test/fixture/test_main.c new file mode 100644 index 0000000..adbe919 --- /dev/null +++ b/test/fixture/test_main.c @@ -0,0 +1,4 @@ +int main( int argc, char* argv[] ) +{ + return 0; +} -- cgit v0.12 From 043f3fe3c182730eca17d68030f1de34fa88477e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Feb 2019 14:42:01 -0800 Subject: Fix Issue #3303 --config=force overwritting passed in Environment's Decider and not cleaning it up when configure context is complete --- src/CHANGES.txt | 2 + src/engine/SCons/Environment.py | 1 + src/engine/SCons/SConf.py | 81 +++++++++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 4acfcba..8fccdeb 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -14,6 +14,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER The Configure logic directly calls the decider when using --config=force but wasn't handling that exception. This would yield minimally configure tests using TryLink() not running and leaving TypeError Nonetype exception in config.log + - Fix Issue #3303 - Handle --config=force overwriting the Environment passed into Configure()'s + Decider and not clearing it when the configure context is completed. From Daniel Moody: - Change the default for AppendENVPath to delete_existing=0, so path diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 152f0bc..eea8aed 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -2372,6 +2372,7 @@ class OverrideEnvironment(Base): kw = copy_non_reserved_keywords(kw) self.__dict__['overrides'].update(semi_deepcopy(kw)) + # The entry point that will be used by the external world # to refer to a construction environment. This allows the wrapper # interface to extend a construction environment for its own purposes diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index f23c401..b123c11 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -324,24 +324,6 @@ class SConfBuildTask(SCons.Taskmaster.AlwaysTask): s = sys.stdout = sys.stderr = Streamer(sys.stdout) try: env = self.targets[0].get_build_env() - if cache_mode == FORCE: - # Set up the Decider() to force rebuilds by saying - # that every source has changed. Note that we still - # call the environment's underlying source decider so - # that the correct .sconsign info will get calculated - # and keep the build state consistent. - def force_build(dependency, target, prev_ni, - env_decider=env.decide_source, - node=None): - try: - env_decider(dependency, target, prev_ni) - except DeciderNeedsNode as e: - e.decider(target, prev_ni, node=target) - except Exception as e: - raise e - return True - if env.decide_source.__code__ is not force_build.__code__: - env.Decider(force_build) env['PSTDOUT'] = env['PSTDERR'] = s try: sconf.cached = 0 @@ -412,12 +394,42 @@ class SConfBase(object): build tests in the VariantDir, not in the SourceDir) """ global SConfFS + + # Now create isolated override so setting source_decider doesn't affect parent Environment + if cache_mode == FORCE: + self.original_env = env + self.env = env.Clone() + + # Set up the Decider() to force rebuilds by saying + # that every source has changed. Note that we still + # call the environment's underlying source decider so + # that the correct .sconsign info will get calculated + # and keep the build state consistent. + def force_build(dependency, target, prev_ni, + env_decider=env.decide_source, + node=None): + try: + env_decider(dependency, target, prev_ni) + except DeciderNeedsNode as e: + e.decider(target, prev_ni, node=target) + except Exception as e: + raise e + return True + + if self.env.decide_source.__code__ is not force_build.__code__: + self.env.Decider(force_build) + + else: + self.env = env + + # print("Override env:%s"%env) + if not SConfFS: SConfFS = SCons.Node.FS.default_fs or \ SCons.Node.FS.FS(env.fs.pathTop) if sconf_global is not None: raise SCons.Errors.UserError - self.env = env + if log_file is not None: log_file = SConfFS.File(env.subst(log_file)) self.logfile = log_file @@ -456,6 +468,7 @@ class SConfBase(object): env = sconf.Finish() """ self._shutdown() + return self.env def Define(self, name, value = None, comment = None): @@ -510,6 +523,20 @@ class SConfBase(object): n.attributes = SCons.Node.Node.Attrs() n.attributes.keep_targetinfo = 1 + if True: + # Some checkers have intermediate files (for example anything that compiles a c file into a program to run + # Those files need to be set to not release their target info, otherwise taskmaster will throw a + # Nonetype not callable + for c in n.children(scan=False): + # Keep debug code here. + # print("Checking [%s] for builders and then setting keep_targetinfo"%c) + if c.has_builder(): + n.store_info = 0 + if not hasattr(c, 'attributes'): + c.attributes = SCons.Node.Node.Attrs() + c.attributes.keep_targetinfo = 1 + # pass + ret = 1 try: @@ -746,10 +773,18 @@ class SConfBase(object): self.logstream.write("\n") self.logstream.close() self.logstream = None - # remove the SConfSourceBuilder from the environment - blds = self.env['BUILDERS'] - del blds['SConfSourceBuilder'] - self.env.Replace( BUILDERS=blds ) + + # Now reset the decider if we changed it due to --config=force + # We saved original Environment passed in and cloned it to isolate + # it from being changed. + if cache_mode == FORCE: + self.env.Decider(self.original_env.decide_source) + + # remove the SConfSourceBuilder from the environment + blds = self.env['BUILDERS'] + del blds['SConfSourceBuilder'] + self.env.Replace( BUILDERS=blds ) + self.active = 0 sconf_global = None if not self.config_h is None: -- cgit v0.12 From 32201ba249861390fae34198eedd787972db09ea Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Wed, 27 Feb 2019 11:50:47 +0100 Subject: Do not store build host name if reproducible builds are wanted. See https://reproducible-builds.org/ for why this is good. This affected scons itself, which differed in the line __buildsys__ = "..." --- SConstruct | 8 ++++++-- src/CHANGES.txt | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index 8f7ceaf..760b1c6 100644 --- a/SConstruct +++ b/SConstruct @@ -84,7 +84,10 @@ if not developer: build_system = ARGUMENTS.get('BUILD_SYSTEM') if not build_system: - build_system = socket.gethostname().split('.')[0] + if os.environ.get('SOURCE_DATE_EPOCH'): + build_system = '_reproducible' + else: + build_system = socket.gethostname().split('.')[0] version = ARGUMENTS.get('VERSION', '') if not version: @@ -159,7 +162,8 @@ command_line_variables = [ ("BUILD_SYSTEM=", "The system on which the packages were built. " + "The default is whatever hostname is returned " + - "by socket.gethostname()."), + "by socket.gethostname(). If SOURCE_DATE_EPOCH " + + "env var is set, '_reproducible' is the default."), ("CHECKPOINT=", "The specific checkpoint release being packaged, " + "which will be appended to the VERSION string. " + diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 0d02ec0..39b11f2 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -26,6 +26,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Add the textfile tool to the default tool list - Fix syntax on is/is not cluases: should not use with a literal + From Bernhard M. Wiedemann: + - Do not store build host name if reproducible builds are wanted + RELEASE 3.0.4 - Mon, 20 Jan 2019 22:49:27 +0000 -- cgit v0.12 From 46fe4773a2d063ad9bf74ff3aef258da102865d8 Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Thu, 28 Feb 2019 16:23:25 +0100 Subject: Do not store build user name --- SConstruct | 6 +++++- src/CHANGES.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index 760b1c6..b94a3e3 100644 --- a/SConstruct +++ b/SConstruct @@ -81,6 +81,8 @@ if not developer: developer = os.environ.get(variable) if developer: break + if os.environ.get('SOURCE_DATE_EPOCH'): + developer = '_reproducible' build_system = ARGUMENTS.get('BUILD_SYSTEM') if not build_system: @@ -181,7 +183,9 @@ command_line_variables = [ ("DEVELOPER=", "The developer who created the packages. " + "The default is the first set environment " + - "variable from the list $USERNAME, $LOGNAME, $USER."), + "variable from the list $USERNAME, $LOGNAME, $USER." + + "If the SOURCE_DATE_EPOCH env var is set, " + + "'_reproducible' is the default."), ("REVISION=", "The revision number of the source being built. " + "The default is the git hash returned " + diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 39b11f2..b9052bb 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -27,7 +27,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fix syntax on is/is not cluases: should not use with a literal From Bernhard M. Wiedemann: - - Do not store build host name if reproducible builds are wanted + - Do not store build host+user name if reproducible builds are wanted RELEASE 3.0.4 - Mon, 20 Jan 2019 22:49:27 +0000 -- cgit v0.12 From 060f9646633146fb9c64d2a31e9c3f3117c2f85b Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Thu, 28 Feb 2019 16:31:41 +0100 Subject: Fix typo --- src/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index b9052bb..1eb08db 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -24,7 +24,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Quiet open file ResourceWarnings on Python >= 3.6 caused by not using a context manager around Popen.stdout - Add the textfile tool to the default tool list - - Fix syntax on is/is not cluases: should not use with a literal + - Fix syntax on is/is not clauses: should not use with a literal From Bernhard M. Wiedemann: - Do not store build host+user name if reproducible builds are wanted -- cgit v0.12 From 910af82db286c12414b2abc252c484bcaaa9beb4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 28 Feb 2019 11:20:58 -0500 Subject: Update SConf tests to add scan arg to mocked out Node children() method since this is now used by SConf's logic --- src/engine/SCons/SConfTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index cf8a7fb..ff544f5 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -194,7 +194,7 @@ class SConfTestCase(unittest.TestCase): pass def add_post_action(self, *actions): pass - def children(self): + def children(self, scan = 1): return [] def get_state(self): return self.state -- cgit v0.12 From 2ea6b7a551620c56f61a5dd9a9c94e342993795e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 28 Feb 2019 13:21:58 -0500 Subject: Add debug logic to try to find why test/TEX/variant_dir.py is failing not finding dvipdf --- .travis/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/install.sh b/.travis/install.sh index c1a4d02..b4b9ef1 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -47,4 +47,6 @@ else tar xzf rel-3.0.12.tar.gz cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. fi + + which dvipdf fi -- cgit v0.12 From 7b192e7b4c92606ffe1b07adbb82bda039a17975 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 28 Feb 2019 23:16:12 -0600 Subject: only use no-unistd option with MSVC environment --- src/CHANGES.txt | 4 +++- src/engine/SCons/Tool/lex.py | 10 ++++++---- src/engine/SCons/Tool/msvc.py | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index bfea661..7393664 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -21,7 +21,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fixed bug which threw error when running SCons on windows system with no MSVC installed. - Update link tool to convert target to node before accessing node member - Update mingw tool to remove MSVC like nologo CCFLAG - - Update lex tool to work in mingw environments + - Add default paths for lex tool on windows + - Add lex construction variable LEXUNISTD for turning off unix headers on windows + - Update lex tool to use win_flex on windows if available From Mats Wichmann: - Quiet open file ResourceWarnings on Python >= 3.6 caused by diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index 3506f7c..a63ddc9 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -75,7 +75,7 @@ def get_lex_path(env, append_paths=False): """ # save existing path to reset if we don't want to append any paths envPath = env['ENV']['PATH'] - bins = ['win_flex', 'lex', 'flex'] + bins = ['flex', 'lex', 'win_flex'] for prog in bins: bin_path = SCons.Tool.find_program_path( @@ -110,14 +110,16 @@ def generate(env): cxx_file.add_action(".ll", LexAction) cxx_file.add_emitter(".ll", lexEmitter) + env["LEXFLAGS"] = SCons.Util.CLVar("") + if sys.platform == 'win32': get_lex_path(env, append_paths=True) - env["LEX"] = env.Detect(['win_flex', 'lex', 'flex']) - env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd") + env["LEX"] = env.Detect(['flex', 'lex', 'win_flex']) + if not env.get("LEXUNISTD"): + env["LEXUNISTD"] = SCons.Util.CLVar("") env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET" else: env["LEX"] = env.Detect(["flex", "lex"]) - env["LEXFLAGS"] = SCons.Util.CLVar("") env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" def exists(env): diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index 9f3c1fa..2352088 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -271,6 +271,10 @@ def generate(env): env['SHOBJPREFIX'] = '$OBJPREFIX' env['SHOBJSUFFIX'] = '$OBJSUFFIX' + # MSVC probably wont support unistd.h so default + # without it for lex generation + env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd") + # Set-up ms tools paths msvc_setup_env_once(env) -- cgit v0.12 From f765b5c4fa6b41de214b243daa6dd3ad6356dfe2 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 12 Feb 2019 09:42:43 -0700 Subject: Drop -tt flag in runtest.py Test runs launch Python with the -tt flag to error on inconsistent tab usage. That flag is no longer part of Python 3, though it is silently accepted and ignored in cpython. In PyPy3, it errors. We have other ways to detect such things now anyway, so just drop. Signed-off-by: Mats Wichmann --- runtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtest.py b/runtest.py index 9ffe374..eecf19d 100755 --- a/runtest.py +++ b/runtest.py @@ -797,7 +797,7 @@ tests_failing = 0 def run_test(t, io_lock, run_async=True): global tests_completed, tests_passing, tests_failing header = "" - command_args = ['-tt'] + command_args = [] if debug: command_args.append(debug) command_args.append(t.path) -- cgit v0.12 From 83c4b8ff692066bf29daa9a6aebf10818a914475 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 1 Mar 2019 09:46:54 -0700 Subject: [WIP] for #3304: drop use of -tt if py3 As noted in issue #3304, the Python flag to error on inconsistent tabs/spaces has been dropped for Python 3 interpreters; while CPython still accepts it, silently ignoring it, PyPy3 errors out. This change adds the flag throughout the testsuite run only if using a Python2 interpreter. Signed-off-by: Mats Wichmann --- SConstruct | 6 ------ runtest.py | 2 ++ test/runtest/baseline/combined.py | 7 ++++--- test/runtest/baseline/fail.py | 3 ++- test/runtest/baseline/no_result.py | 3 ++- test/runtest/baseline/pass.py | 3 ++- test/runtest/fallback.py | 7 ++++--- test/runtest/noqmtest.py | 7 ++++--- test/runtest/print_time.py | 7 ++++--- test/runtest/python.py | 7 ++++++- test/runtest/simple/combined.py | 7 ++++--- test/runtest/simple/fail.py | 3 ++- test/runtest/simple/no_result.py | 3 ++- test/runtest/simple/pass.py | 3 ++- test/runtest/src.py | 5 +++-- test/runtest/testargv.py | 5 +++-- test/runtest/testlistfile.py | 3 ++- test/runtest/xml/output.py | 7 ++++--- testing/framework/TestRuntest.py | 10 ++++++++-- testing/framework/TestSCons.py | 4 +++- testing/framework/TestSCons_time.py | 4 +++- 21 files changed, 66 insertions(+), 40 deletions(-) diff --git a/SConstruct b/SConstruct index b94a3e3..5ac5692 100644 --- a/SConstruct +++ b/SConstruct @@ -46,8 +46,6 @@ import time import socket import textwrap - - import bootstrap project = 'scons' @@ -56,7 +54,6 @@ copyright = "Copyright (c) %s The SCons Foundation" % copyright_years SConsignFile() - # # We let the presence or absence of various utilities determine whether # or not we bother to build certain pieces of things. This should allow @@ -129,9 +126,6 @@ if build_id is None: else: build_id = '' - -python_ver = sys.version[0:3] - # # Adding some paths to sys.path, this is mainly needed # for the doc toolchain. diff --git a/runtest.py b/runtest.py index eecf19d..ccc170a 100755 --- a/runtest.py +++ b/runtest.py @@ -798,6 +798,8 @@ def run_test(t, io_lock, run_async=True): global tests_completed, tests_passing, tests_failing header = "" command_args = [] + if sys.version_info[0] < 3: + command_args.append('-tt') if debug: command_args.append(debug) command_args.append(t.path) diff --git a/test/runtest/baseline/combined.py b/test/runtest/baseline/combined.py index 35c1796..228d42d 100644 --- a/test/runtest/baseline/combined.py +++ b/test/runtest/baseline/combined.py @@ -34,6 +34,7 @@ import os import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test_fail_py = os.path.join('test', 'fail.py') test_no_result_py = os.path.join('test', 'no_result.py') test_pass_py = os.path.join('test', 'pass.py') @@ -49,11 +50,11 @@ test.write_no_result_test(['test', 'no_result.py']) test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(pythonstring)s -tt %(test_fail_py)s +%(pythonstring)s%(pythonflags)s %(test_fail_py)s FAILING TEST STDOUT -%(pythonstring)s -tt %(test_no_result_py)s +%(pythonstring)s%(pythonflags)s %(test_no_result_py)s NO RESULT TEST STDOUT -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT Failed the following test: diff --git a/test/runtest/baseline/fail.py b/test/runtest/baseline/fail.py index 5687160..e2aff4a 100644 --- a/test/runtest/baseline/fail.py +++ b/test/runtest/baseline/fail.py @@ -31,6 +31,7 @@ Test how we handle a failing test specified on the command line. import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -39,7 +40,7 @@ test.subdir('test') test.write_failing_test(['test', 'fail.py']) expect_stdout = """\ -%(pythonstring)s -tt test/fail.py +%(pythonstring)s%(pythonflags)s test/fail.py FAILING TEST STDOUT """ % locals() diff --git a/test/runtest/baseline/no_result.py b/test/runtest/baseline/no_result.py index 2149594..b63d0c6 100644 --- a/test/runtest/baseline/no_result.py +++ b/test/runtest/baseline/no_result.py @@ -31,6 +31,7 @@ Test how we handle a no-results test specified on the command line. import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -39,7 +40,7 @@ test.subdir('test') test.write_no_result_test(['test', 'no_result.py']) expect_stdout = """\ -%(pythonstring)s -tt test/no_result.py +%(pythonstring)s%(pythonflags)s test/no_result.py NO RESULT TEST STDOUT """ % locals() diff --git a/test/runtest/baseline/pass.py b/test/runtest/baseline/pass.py index affa486..481fc97 100644 --- a/test/runtest/baseline/pass.py +++ b/test/runtest/baseline/pass.py @@ -33,6 +33,7 @@ import os import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test_pass_py = os.path.join('test', 'pass.py') test = TestRuntest.TestRuntest() @@ -42,7 +43,7 @@ test.subdir('test') test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT """ % locals() diff --git a/test/runtest/fallback.py b/test/runtest/fallback.py index 1229b28..b137307 100644 --- a/test/runtest/fallback.py +++ b/test/runtest/fallback.py @@ -34,6 +34,7 @@ import os import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -56,11 +57,11 @@ test.write_no_result_test(test_no_result_py) test.write_passing_test(test_pass_py) expect_stdout = """\ -%(pythonstring)s -tt %(test_fail_py)s +%(pythonstring)s%(pythonflags)s %(test_fail_py)s FAILING TEST STDOUT -%(pythonstring)s -tt %(test_no_result_py)s +%(pythonstring)s%(pythonflags)s %(test_no_result_py)s NO RESULT TEST STDOUT -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT Failed the following test: diff --git a/test/runtest/noqmtest.py b/test/runtest/noqmtest.py index cea2f11..fcf7ac0 100644 --- a/test/runtest/noqmtest.py +++ b/test/runtest/noqmtest.py @@ -34,6 +34,7 @@ import os import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -48,11 +49,11 @@ test.write_no_result_test(test_no_result_py) test.write_passing_test(test_pass_py) expect_stdout = """\ -%(pythonstring)s -tt %(test_fail_py)s +%(pythonstring)s%(pythonflags)s %(test_fail_py)s FAILING TEST STDOUT -%(pythonstring)s -tt %(test_no_result_py)s +%(pythonstring)s%(pythonflags)s %(test_no_result_py)s NO RESULT TEST STDOUT -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT Failed the following test: diff --git a/test/runtest/print_time.py b/test/runtest/print_time.py index 244c6f8..322b88b 100644 --- a/test/runtest/print_time.py +++ b/test/runtest/print_time.py @@ -36,6 +36,7 @@ import TestCmd import TestRuntest pythonstring = re.escape(TestRuntest.pythonstring) +pythonflags = TestRuntest.pythonflags test_fail_py = re.escape(os.path.join('test', 'fail.py')) test_no_result_py = re.escape(os.path.join('test', 'no_result.py')) test_pass_py = re.escape(os.path.join('test', 'pass.py')) @@ -51,13 +52,13 @@ test.write_no_result_test(['test', 'no_result.py']) test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(pythonstring)s -tt %(test_fail_py)s +%(pythonstring)s%(pythonflags)s %(test_fail_py)s FAILING TEST STDOUT Test execution time: \\d+.\\d seconds -%(pythonstring)s -tt %(test_no_result_py)s +%(pythonstring)s%(pythonflags)s %(test_no_result_py)s NO RESULT TEST STDOUT Test execution time: \\d+.\\d seconds -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT Test execution time: \\d+.\\d seconds Total execution time for all tests: \\d+.\\d seconds diff --git a/test/runtest/python.py b/test/runtest/python.py index bcbc062..14156e0 100644 --- a/test/runtest/python.py +++ b/test/runtest/python.py @@ -43,6 +43,11 @@ test_pass_py = os.path.join('test', 'pass.py') head, python = os.path.split(TestRuntest.python) head, dir = os.path.split(head) +# deciding whether or not to use -tt flag here is dicey. +# we'll go ahead and use TestRuntest.pythonflags, but it uses the +# python version then in use, which could be different +pythonflags = TestRuntest.pythonflags + # We have to normalize the python path here, because some installations don't like # getting called with "/bin/../bin/python" as first argument, e.g. Fedora 17 Desktop. mypython = os.path.normpath(os.path.join(head, dir, os.path.pardir, dir, python)) @@ -60,7 +65,7 @@ test.subdir('test') test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(mypythonstring)s -tt %(test_pass_py)s +%(mypythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT """ % locals() diff --git a/test/runtest/simple/combined.py b/test/runtest/simple/combined.py index 616f4d5..ec0a1bb 100644 --- a/test/runtest/simple/combined.py +++ b/test/runtest/simple/combined.py @@ -37,6 +37,7 @@ import TestRuntest test = TestRuntest.TestRuntest() pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test_fail_py = os.path.join('test', 'fail.py') test_no_result_py = os.path.join('test', 'no_result.py') test_pass_py = os.path.join('test', 'pass.py') @@ -50,11 +51,11 @@ test.write_no_result_test(['test', 'no_result.py']) test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(pythonstring)s -tt %(test_fail_py)s +%(pythonstring)s%(pythonflags)s %(test_fail_py)s FAILING TEST STDOUT -%(pythonstring)s -tt %(test_no_result_py)s +%(pythonstring)s%(pythonflags)s %(test_no_result_py)s NO RESULT TEST STDOUT -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT Failed the following test: diff --git a/test/runtest/simple/fail.py b/test/runtest/simple/fail.py index 8b800fb..f26f00e 100644 --- a/test/runtest/simple/fail.py +++ b/test/runtest/simple/fail.py @@ -31,6 +31,7 @@ Test how we handle a failing test specified on the command line. import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -39,7 +40,7 @@ test.subdir('test') test.write_failing_test(['test', 'fail.py']) expect_stdout = """\ -%(pythonstring)s -tt test/fail.py +%(pythonstring)s%(pythonflags)s test/fail.py FAILING TEST STDOUT """ % locals() diff --git a/test/runtest/simple/no_result.py b/test/runtest/simple/no_result.py index 91af7e4..2fd40b4 100644 --- a/test/runtest/simple/no_result.py +++ b/test/runtest/simple/no_result.py @@ -31,6 +31,7 @@ Test how we handle a no-results test specified on the command line. import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -39,7 +40,7 @@ test.subdir('test') test.write_no_result_test(['test', 'no_result.py']) expect_stdout = """\ -%(pythonstring)s -tt test/no_result.py +%(pythonstring)s%(pythonflags)s test/no_result.py NO RESULT TEST STDOUT """ % locals() diff --git a/test/runtest/simple/pass.py b/test/runtest/simple/pass.py index 6e5b6b0..7ceb9a0 100644 --- a/test/runtest/simple/pass.py +++ b/test/runtest/simple/pass.py @@ -31,6 +31,7 @@ Test how we handle a passing test specified on the command line. import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test = TestRuntest.TestRuntest() @@ -39,7 +40,7 @@ test.subdir('test') test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ -%(pythonstring)s -tt test/pass.py +%(pythonstring)s%(pythonflags)s test/pass.py PASSING TEST STDOUT """ % locals() diff --git a/test/runtest/src.py b/test/runtest/src.py index 23894f9..cbce2bd 100644 --- a/test/runtest/src.py +++ b/test/runtest/src.py @@ -39,6 +39,7 @@ test.subdir(['src'], ['src', 'suite']) pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags src_passTests_py = os.path.join('src', 'passTests.py') src_suite_passTests_py = os.path.join('src', 'suite', 'passTests.py') @@ -51,9 +52,9 @@ test.write_passing_test(['src', 'suite', 'pass.py']) test.write_passing_test(['src', 'suite', 'passTests.py']) expect_stdout = """\ -%(pythonstring)s -tt %(src_passTests_py)s +%(pythonstring)s%(pythonflags)s %(src_passTests_py)s PASSING TEST STDOUT -%(pythonstring)s -tt %(src_suite_passTests_py)s +%(pythonstring)s%(pythonflags)s %(src_suite_passTests_py)s PASSING TEST STDOUT """ % locals() diff --git a/test/runtest/testargv.py b/test/runtest/testargv.py index 62faf51..22e57e8 100644 --- a/test/runtest/testargv.py +++ b/test/runtest/testargv.py @@ -40,6 +40,7 @@ test.subdir('test', ['test', 'subdir']) files = {} files['pythonstring'] = TestRuntest.pythonstring +files['pythonflags'] = TestRuntest.pythonflags files['one'] = os.path.join('test/subdir', 'test_one.py') files['two'] = os.path.join('test/subdir', 'two.py') @@ -50,9 +51,9 @@ test.write_passing_test(files['two']) test.write_passing_test(files['three']) expect_stdout = """\ -%(pythonstring)s -tt %(one)s +%(pythonstring)s%(pythonflags)s %(one)s PASSING TEST STDOUT -%(pythonstring)s -tt %(two)s +%(pythonstring)s%(pythonflags)s %(two)s PASSING TEST STDOUT """ % files diff --git a/test/runtest/testlistfile.py b/test/runtest/testlistfile.py index b86b0f2..063a8d0 100644 --- a/test/runtest/testlistfile.py +++ b/test/runtest/testlistfile.py @@ -34,6 +34,7 @@ import re import TestRuntest pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags test_fail_py = os.path.join('test', 'fail.py') test_no_result_py = os.path.join('test', 'no_result.py') test_pass_py = os.path.join('test', 'pass.py') @@ -54,7 +55,7 @@ test.write('t.txt', """\ """ % locals()) expect_stdout = """\ -%(pythonstring)s -tt %(test_pass_py)s +%(pythonstring)s%(pythonflags)s %(test_pass_py)s PASSING TEST STDOUT """ % locals() diff --git a/test/runtest/xml/output.py b/test/runtest/xml/output.py index e669a3a..88bca04 100644 --- a/test/runtest/xml/output.py +++ b/test/runtest/xml/output.py @@ -38,6 +38,7 @@ test = TestRuntest.TestRuntest(match = TestCmd.match_re, diff = TestCmd.diff_re) pythonstring = re.escape(TestRuntest.pythonstring) +pythonflags = TestRuntest.pythonflags test_fail_py = re.escape(os.path.join('test', 'fail.py')) test_no_result_py = re.escape(os.path.join('test', 'no_result.py')) test_pass_py = re.escape(os.path.join('test', 'pass.py')) @@ -58,7 +59,7 @@ expect = """\ %(test_fail_py)s - %(pythonstring)s -tt %(test_fail_py)s + %(pythonstring)s%(pythonflags)s %(test_fail_py)s 1 FAILING TEST STDOUT @@ -68,7 +69,7 @@ expect = """\ %(test_no_result_py)s - %(pythonstring)s -tt %(test_no_result_py)s + %(pythonstring)s%(pythonflags)s %(test_no_result_py)s 2 NO RESULT TEST STDOUT @@ -78,7 +79,7 @@ expect = """\ %(test_pass_py)s - %(pythonstring)s -tt %(test_pass_py)s + %(pythonstring)s%(pythonflags)s %(test_pass_py)s 0 PASSING TEST STDOUT diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py index 7b3bb52..faf5d60 100644 --- a/testing/framework/TestRuntest.py +++ b/testing/framework/TestRuntest.py @@ -27,6 +27,7 @@ from TestCommon import __all__ __all__.extend([ 'TestRuntest', 'pythonstring', + 'pythonflags', ]) if re.search('\s', python): @@ -34,7 +35,9 @@ if re.search('\s', python): else: pythonstring = python pythonstring = pythonstring.replace('\\', '\\\\') - +pythonflags = '' +if sys.version_info[0] < 3: + pythonflags = ' -tt' failing_test_template = """\ import sys @@ -104,7 +107,10 @@ class TestRuntest(TestCommon): if 'program' not in kw: kw['program'] = 'runtest.py' if 'interpreter' not in kw: - kw['interpreter'] = [python, '-tt'] + kw['interpreter'] = [python,] + if sys.version_info[0] < 3: + kw['interpreter'].append('-tt') + if 'match' not in kw: kw['match'] = match_exact if 'workdir' not in kw: diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 7fe48e8..57b300d 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -247,7 +247,9 @@ class TestSCons(TestCommon): elif not self.external and not os.path.isabs(kw['program']): kw['program'] = os.path.join(self.orig_cwd, kw['program']) if 'interpreter' not in kw and not os.environ.get('SCONS_EXEC'): - kw['interpreter'] = [python, '-tt'] + kw['interpreter'] = [python,] + if sys.version_info[0] < 3: + kw['interpreter'].append('-tt') if 'match' not in kw: kw['match'] = match_exact if 'workdir' not in kw: diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index 6299f51..8260bef 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -186,7 +186,9 @@ class TestSCons_time(TestCommon): kw['program'] = p if 'interpreter' not in kw: - kw['interpreter'] = [python, '-tt'] + kw['interpreter'] = [python,] + if sys.version_info[0] < 3: + kw['interpreter'].append('-tt') if 'match' not in kw: kw['match'] = match_exact -- cgit v0.12 From 4d40813e369694fbe3cfe5488178bbb722d67319 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 2 Mar 2019 10:45:34 -0700 Subject: [PYPY3] [PY 3.8] CacheDir tests use context managers Python 3.8 complains about unclosed files on nearly all of the CacheDir tests. PyPy3 fails 8 of the 20 tests for the same reason, though since it's based on an earlier version of Python (3.5 at the moment) it does not display the resourcewarning messages. Update sequences of open().write() which are the primary problem area to use context managers for automatic closing, and for consistency, the rest of the open/read/write stuff as well. With this change, all these tests pass in PyPy3. Python 3.8 does not, as it is spewing other warning messages which also end up in the stderr, which is fatal to this set of tests, but it has quieted the warnings from the CacheDir tests themselves. Signed-off-by: Mats Wichmann --- test/CacheDir/CacheDir.py | 11 ++++++----- test/CacheDir/SideEffect.py | 7 ++++--- test/CacheDir/VariantDir.py | 11 ++++++----- test/CacheDir/debug.py | 11 ++++++----- test/CacheDir/environment.py | 11 ++++++----- test/CacheDir/multi-targets.py | 6 ++++-- test/CacheDir/multiple-targets.py | 6 ++++-- test/CacheDir/option--cd.py | 11 ++++++----- test/CacheDir/option--cf.py | 11 ++++++----- test/CacheDir/option--cr.py | 11 ++++++----- test/CacheDir/option--cs.py | 22 ++++++++++++---------- test/CacheDir/scanner-target.py | 5 ++--- test/CacheDir/source-scanner.py | 5 ++--- 13 files changed, 70 insertions(+), 58 deletions(-) diff --git a/test/CacheDir/CacheDir.py b/test/CacheDir/CacheDir.py index 3d2c3b5..3b3e72b 100644 --- a/test/CacheDir/CacheDir.py +++ b/test/CacheDir/CacheDir.py @@ -53,11 +53,12 @@ SConscript('SConscript') test.write(['src', 'SConscript'], """\ def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/SideEffect.py b/test/CacheDir/SideEffect.py index 4eae4c3..3242e78 100644 --- a/test/CacheDir/SideEffect.py +++ b/test/CacheDir/SideEffect.py @@ -39,15 +39,16 @@ cache = test.workpath('cache') test.write(['work', 'SConstruct'], """\ DefaultEnvironment(tools=[]) def copy(source, target): - open(target, "w").write(open(source, "r").read()) + with open(target, "w") as f, open(source, "r") as f2: + f.write(f2.read()) def build(env, source, target): s = str(source[0]) t = str(target[0]) copy(s, t) if target[0].side_effects: - side_effect = open(str(target[0].side_effects[0]), "a") - side_effect.write(s + ' -> ' + t + '\\n') + with open(str(target[0].side_effects[0]), "a") as side_effect: + side_effect.write(s + ' -> ' + t + '\\n') CacheDir(r'%(cache)s') diff --git a/test/CacheDir/VariantDir.py b/test/CacheDir/VariantDir.py index 58918be..2c3d73f 100644 --- a/test/CacheDir/VariantDir.py +++ b/test/CacheDir/VariantDir.py @@ -42,11 +42,12 @@ cat_out = test.workpath('cat.out') test.write(['src', 'SConscript'], """\ def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/debug.py b/test/CacheDir/debug.py index 7e08e0b..16f4702 100644 --- a/test/CacheDir/debug.py +++ b/test/CacheDir/debug.py @@ -53,11 +53,12 @@ SConscript('SConscript') test.write(['src', 'SConscript'], """\ def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/environment.py b/test/CacheDir/environment.py index 5d8eb6c..1024ce0 100644 --- a/test/CacheDir/environment.py +++ b/test/CacheDir/environment.py @@ -54,11 +54,12 @@ SConscript('SConscript') test.write(['src', 'SConscript'], """\ def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env_cache = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env_nocache = env_cache.Clone() env_nocache.CacheDir(None) diff --git a/test/CacheDir/multi-targets.py b/test/CacheDir/multi-targets.py index 7977ba0..fa6e8d1 100644 --- a/test/CacheDir/multi-targets.py +++ b/test/CacheDir/multi-targets.py @@ -42,8 +42,10 @@ multiple_foo = test.workpath('multiple', 'foo') test.write(['multiple', 'SConstruct'], """\ DefaultEnvironment(tools=[]) def touch(env, source, target): - open('foo', 'w').write("") - open('bar', 'w').write("") + with open('foo', 'w') as f: + f.write("") + with open('bar', 'w') as f: + f.write("") CacheDir(r'%(cache)s') env = Environment(tools=[]) env.Command(['foo', 'bar'], ['input'], touch) diff --git a/test/CacheDir/multiple-targets.py b/test/CacheDir/multiple-targets.py index 9f94e4c..99ab8da 100644 --- a/test/CacheDir/multiple-targets.py +++ b/test/CacheDir/multiple-targets.py @@ -40,8 +40,10 @@ test.subdir('cache') test.write('SConstruct', """\ DefaultEnvironment(tools=[]) def touch(env, source, target): - open('foo', 'w').write("") - open('bar', 'w').write("") + with open('foo', 'w') as f: + f.write("") + with open('bar', 'w') as f: + f.write("") CacheDir(r'%s') env = Environment(tools=[], ) env.Command(['foo', 'bar'], ['input'], touch) diff --git a/test/CacheDir/option--cd.py b/test/CacheDir/option--cd.py index 20d1184..1620858 100644 --- a/test/CacheDir/option--cd.py +++ b/test/CacheDir/option--cd.py @@ -42,11 +42,12 @@ test.write(['src', 'SConstruct'], """ DefaultEnvironment(tools=[]) def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/option--cf.py b/test/CacheDir/option--cf.py index 5e823ae..2d51e03 100644 --- a/test/CacheDir/option--cf.py +++ b/test/CacheDir/option--cf.py @@ -41,11 +41,12 @@ test.write(['src', 'SConstruct'], """ DefaultEnvironment(tools=[]) def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/option--cr.py b/test/CacheDir/option--cr.py index 4ed587c..b7696c5 100644 --- a/test/CacheDir/option--cr.py +++ b/test/CacheDir/option--cr.py @@ -42,11 +42,12 @@ test.write(['src', 'SConstruct'], """ DefaultEnvironment(tools=[]) def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/CacheDir/option--cs.py b/test/CacheDir/option--cs.py index b73fb70..feb89bd 100644 --- a/test/CacheDir/option--cs.py +++ b/test/CacheDir/option--cs.py @@ -45,11 +45,12 @@ test.subdir('cache', 'src1', 'src2') test.write(['src1', 'build.py'], r""" import sys -open('cat.out', 'a').write(sys.argv[1] + "\n") -file = open(sys.argv[1], 'w') -for src in sys.argv[2:]: - file.write(open(src, 'r').read()) -file.close() +with open('cat.out', 'a') as f: + f.write(sys.argv[1] + "\n") +with open(sys.argv[1], 'w') as f: + for src in sys.argv[2:]: + with open(src, 'r') as f2: + f.write(f2.read()) """) cache = test.workpath('cache') @@ -58,11 +59,12 @@ test.write(['src1', 'SConstruct'], """ DefaultEnvironment(tools=[]) def cat(env, source, target): target = str(target[0]) - open('cat.out', 'a').write(target + "\\n") - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(tools=[], BUILDERS={'Internal':Builder(action=cat), 'External':Builder(action=r'%(_python_)s build.py $TARGET $SOURCES')}) diff --git a/test/CacheDir/scanner-target.py b/test/CacheDir/scanner-target.py index 7df9792..249e587 100644 --- a/test/CacheDir/scanner-target.py +++ b/test/CacheDir/scanner-target.py @@ -48,9 +48,8 @@ CacheDir(r'%s') def docopy(target,source,env): data = source[0].get_contents() - f = open(target[0].rfile().get_abspath(), "wb") - f.write(data) - f.close() + with open(target[0].rfile().get_abspath(), "wb") as f: + f.write(data) def sillyScanner(node, env, dirs): print('This is never called (unless we build file.out)') diff --git a/test/CacheDir/source-scanner.py b/test/CacheDir/source-scanner.py index f00360d..1c56499 100644 --- a/test/CacheDir/source-scanner.py +++ b/test/CacheDir/source-scanner.py @@ -50,9 +50,8 @@ CacheDir(r'%(cache)s') def docopy(target,source,env): data = source[0].get_contents() - f = open(target[0].rfile().get_abspath(), "wb") - f.write(data) - f.close() + with open(target[0].rfile().get_abspath(), "wb") as f: + f.write(data) def sillyScanner(node, env, dirs): print('This is never called (unless we build file.out)') -- cgit v0.12 From e6d14fd285273bfa952930ded2c20380f8eec8bc Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 2 Mar 2019 13:49:22 -0700 Subject: [PYPY] [PY 3.8] more context manager use in tests Similar to #3319, add additional use of context managers inside written tests. Fixes som PyPy3 fails and quiets matching warnings from Python 3.8. Signed-off-by: Mats Wichmann --- test/Actions/pre-post.py | 16 ++++++------- test/SideEffect/basic.py | 7 +++--- test/SideEffect/variant_dir.py | 7 +++--- test/VariantDir/errors.py | 51 ++++++++++++++++++++---------------------- 4 files changed, 40 insertions(+), 41 deletions(-) diff --git a/test/Actions/pre-post.py b/test/Actions/pre-post.py index a5acbfb..cd0bfb4 100644 --- a/test/Actions/pre-post.py +++ b/test/Actions/pre-post.py @@ -47,13 +47,11 @@ env = Environment(XXX='bar%(_exe)s') def before(env, target, source): a=str(target[0]) - f=open(a, "wb") - f.write(b"Foo\\n") - f.close() + with open(a, "wb") as f: + f.write(b"Foo\\n") os.chmod(a, os.stat(a)[stat.ST_MODE] | stat.S_IXUSR) - f=open("before.txt", "ab") - f.write((os.path.splitext(str(target[0]))[0] + "\\n").encode()) - f.close() + with open("before.txt", "ab") as f: + f.write((os.path.splitext(str(target[0]))[0] + "\\n").encode()) def after(env, target, source): t = str(target[0]) @@ -106,9 +104,11 @@ test.must_match(['work3', 'dir', 'file'], "build()\n") # work4 start test.write(['work4', 'SConstruct'], """\ def pre_action(target, source, env): - open(str(target[0]), 'ab').write(('pre %%s\\n' %% source[0]).encode()) + with open(str(target[0]), 'ab') as f: + f.write(('pre %%s\\n' %% source[0]).encode()) def post_action(target, source, env): - open(str(target[0]), 'ab').write(('post %%s\\n' %% source[0]).encode()) + with open(str(target[0]), 'ab') as f: + f.write(('post %%s\\n' %% source[0]).encode()) env = Environment() o = env.Command(['pre-post', 'file.out'], 'file.in', diff --git a/test/SideEffect/basic.py b/test/SideEffect/basic.py index af6c264..b5b6381 100644 --- a/test/SideEffect/basic.py +++ b/test/SideEffect/basic.py @@ -37,13 +37,14 @@ test = TestSCons.TestSCons() test.write('SConstruct', """\ def copy(source, target): - open(target, "wb").write(open(source, "rb").read()) + with open(target, "wb") as f, open(source, "rb") as f2: + f.write(f2.read()) def build(env, source, target): copy(str(source[0]), str(target[0])) if target[0].side_effects: - side_effect = open(str(target[0].side_effects[0]), "ab") - side_effect.write(('%%s -> %%s\\n'%%(str(source[0]), str(target[0]))).encode()) + with open(str(target[0].side_effects[0]), "ab") as side_effect: + side_effect.write(('%%s -> %%s\\n'%%(str(source[0]), str(target[0]))).encode()) Build = Builder(action=build) env = Environment(BUILDERS={'Build':Build}, SUBDIR='subdir') diff --git a/test/SideEffect/variant_dir.py b/test/SideEffect/variant_dir.py index 711e426..0e9ae53 100644 --- a/test/SideEffect/variant_dir.py +++ b/test/SideEffect/variant_dir.py @@ -38,13 +38,14 @@ test = TestSCons.TestSCons() test.write('SConstruct', """ def copy(source, target): - open(target, "wb").write(open(source, "rb").read()) + with open(target, "wb") as f, open(source, "rb") as f2: + f.write(f2.read()) def build(env, source, target): copy(str(source[0]), str(target[0])) if target[0].side_effects: - side_effect = open(str(target[0].side_effects[0]), "ab") - side_effect.write(('%s -> %s\\n'%(str(source[0]), str(target[0]))).encode()) + with open(str(target[0].side_effects[0]), "ab") as side_effect: + side_effect.write(('%s -> %s\\n'%(str(source[0]), str(target[0]))).encode()) Build = Builder(action=build) env = Environment(BUILDERS={'Build':Build}) diff --git a/test/VariantDir/errors.py b/test/VariantDir/errors.py index c74d103..d3ecd54 100644 --- a/test/VariantDir/errors.py +++ b/test/VariantDir/errors.py @@ -57,10 +57,10 @@ def fake_scan(node, env, target): def cat(env, source, target): target = str(target[0]) - f = open(target, "w") - for src in source: - f.write(open(str(src), "r").read()) - f.close() + with open(target, "w") as f: + for src in source: + with open(str(src), "r") as f2: + f.write(f2.read()) env = Environment(BUILDERS={'Build':Builder(action=cat)}, SCANNERS=[Scanner(fake_scan, skeys = ['.in'])]) @@ -101,16 +101,15 @@ test.subdir(dir) SConscript = test.workpath(dir, 'SConscript') test.write(SConscript, '') os.chmod(SConscript, os.stat(SConscript)[stat.ST_MODE] & ~stat.S_IWUSR) -f = open(SConscript, 'r') -os.chmod(dir, os.stat(dir)[stat.ST_MODE] & ~stat.S_IWUSR) +with open(SConscript, 'r'): + os.chmod(dir, os.stat(dir)[stat.ST_MODE] & ~stat.S_IWUSR) -test.run(chdir = 'ro-SConscript', - arguments = ".", - status = 2, - stderr = "scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop.\n" % os.path.join('src', 'SConscript')) + test.run(chdir = 'ro-SConscript', + arguments = ".", + status = 2, + stderr = "scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop.\n" % os.path.join('src', 'SConscript')) -os.chmod('ro-SConscript', os.stat('ro-SConscript')[stat.ST_MODE] | stat.S_IWUSR) -f.close() + os.chmod('ro-SConscript', os.stat('ro-SConscript')[stat.ST_MODE] | stat.S_IWUSR) test.run(chdir = 'ro-SConscript', arguments = ".", @@ -129,25 +128,23 @@ test.write([dir, 'SConscript'], '') file_in = test.workpath(dir, 'file.in') test.write(file_in, '') os.chmod(file_in, os.stat(file_in)[stat.ST_MODE] & ~stat.S_IWUSR) -f = open(file_in, 'r') -os.chmod(dir, os.stat(dir)[stat.ST_MODE] & ~stat.S_IWUSR) +with open(file_in, 'r'): + os.chmod(dir, os.stat(dir)[stat.ST_MODE] & ~stat.S_IWUSR) -test.run(chdir = 'ro-src', - arguments = ".", - status = 2, - stderr = """\ -scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. -""" % (os.path.join('src', 'file.in'))) + test.run(chdir = 'ro-src', + arguments = ".", + status = 2, + stderr = """\ + scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. + """ % (os.path.join('src', 'file.in'))) -test.run(chdir = 'ro-src', - arguments = "-k .", - status = 2, - stderr = """\ -scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. + test.run(chdir = 'ro-src', + arguments = "-k .", + status = 2, + stderr = """\ + scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. """ % (os.path.join('src', 'file.in'))) -f.close() - # ensure that specifying multiple source directories for one # build directory results in an error message, rather # than just silently failing. -- cgit v0.12 From e7e70407bd54811604f8e907819e36a6025f7915 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 2 Mar 2019 15:35:59 -0700 Subject: For PR #3320 re-unindent expect strings in with block Previous change created a with: block but accidentally indented two literal strings used as the expected stderr, causing them to no longer match; these are restored to original state. Signed-off-by: Mats Wichmann --- test/VariantDir/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/VariantDir/errors.py b/test/VariantDir/errors.py index d3ecd54..26ef4a2 100644 --- a/test/VariantDir/errors.py +++ b/test/VariantDir/errors.py @@ -135,14 +135,14 @@ with open(file_in, 'r'): arguments = ".", status = 2, stderr = """\ - scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. - """ % (os.path.join('src', 'file.in'))) +scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. +""" % (os.path.join('src', 'file.in'))) test.run(chdir = 'ro-src', arguments = "-k .", status = 2, stderr = """\ - scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. +scons: *** Cannot duplicate `%s' in `build': Permission denied. Stop. """ % (os.path.join('src', 'file.in'))) # ensure that specifying multiple source directories for one -- cgit v0.12 From 701ffd2746fbabe4d1908abbcb4a2d29eaf688f4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 2 Mar 2019 17:27:31 -0800 Subject: copy logic from lex to find win_bison if installed via chocolatey --- src/engine/SCons/Tool/yacc.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/yacc.py b/src/engine/SCons/Tool/yacc.py index cd9b9a8..5db49d3 100644 --- a/src/engine/SCons/Tool/yacc.py +++ b/src/engine/SCons/Tool/yacc.py @@ -41,6 +41,7 @@ import SCons.Tool import SCons.Util from SCons.Platform.mingw import MINGW_DEFAULT_PATHS from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS +from SCons.Platform.win32 import CHOCO_DEFAULT_PATH YaccAction = SCons.Action.Action("$YACCCOM", "$YACCCOMSTR") @@ -97,6 +98,28 @@ def ymEmitter(target, source, env): def yyEmitter(target, source, env): return _yaccEmitter(target, source, env, ['.yy'], '$YACCHXXFILESUFFIX') +def get_yacc_path(env, append_paths=False): + """ + Find the a path containing the lex or flex binaries. If a construction + environment is passed in then append the path to the ENV PATH. + """ + # save existing path to reset if we don't want to append any paths + envPath = env['ENV']['PATH'] + bins = ['bison', 'yacc', 'win_bison'] + + for prog in bins: + bin_path = SCons.Tool.find_program_path( + env, + prog, + default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + if bin_path: + if not append_paths: + env['ENV']['PATH'] = envPath + else: + env.AppendENVPath('PATH', os.path.dirname(bin_path)) + return bin_path + SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') + def generate(env): """Add Builders and construction variables for yacc to an Environment.""" c_file, cxx_file = SCons.Tool.createCFileBuilders(env) @@ -124,7 +147,12 @@ def generate(env): else: SCons.Warnings.Warning('yacc tool requested, but bison binary not found in ENV PATH') - env['YACC'] = env.Detect('bison') or 'yacc' + if sys.platform == 'win32': + get_yacc_path(env, append_paths=True) + env["YACC"] = env.Detect(['bison', 'yacc', 'win_bison']) + else: + env["YACC"] = env.Detect(["bison", "yacc"]) + env['YACCFLAGS'] = SCons.Util.CLVar('') env['YACCCOM'] = '$YACC $YACCFLAGS -o $TARGET $SOURCES' env['YACCHFILESUFFIX'] = '.h' -- cgit v0.12 From 08f5e913d3b70a07672873c4573e236c4ccc7ac1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 2 Mar 2019 20:52:32 -0500 Subject: Add support for usign chocolatey install winflexbison package with tools named win_bison --- src/CHANGES.txt | 3 ++- test/YACC/live-check-output-cleaned.py | 3 ++- test/YACC/live.py | 7 ++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index d4576a1..0a80095 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -16,6 +16,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER leaving TypeError Nonetype exception in config.log - Fix Issue #3303 - Handle --config=force overwriting the Environment passed into Configure()'s Decider and not clearing it when the configure context is completed. + - Add default paths for yacc tool on windows to include cygwin, mingw, and chocolatey From Daniel Moody: - Change the default for AppendENVPath to delete_existing=0, so path @@ -23,7 +24,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fixed bug which threw error when running SCons on windows system with no MSVC installed. - Update link tool to convert target to node before accessing node member - Update mingw tool to remove MSVC like nologo CCFLAG - - Add default paths for lex tool on windows + - Add default paths for lex tool on windows to include cygwin, mingw, and chocolatey - Add lex construction variable LEXUNISTD for turning off unix headers on windows - Update lex tool to use win_flex on windows if available diff --git a/test/YACC/live-check-output-cleaned.py b/test/YACC/live-check-output-cleaned.py index 8329b94..9adaaf0 100644 --- a/test/YACC/live-check-output-cleaned.py +++ b/test/YACC/live-check-output-cleaned.py @@ -34,12 +34,13 @@ _exe = TestSCons._exe test = TestSCons.TestSCons() -yacc = test.where_is('yacc') or test.where_is('bison') +yacc = test.where_is('yacc') or test.where_is('bison') or test.where_is('win_bison') if not yacc: test.skip_test('No yacc or bison found; skipping test.\n') test.write('SConstruct', """ +DefaultEnvironment(tools=[]) foo = Environment(YACCFLAGS='-v -d', tools = ['default', 'yacc']) foo.CFile(source = 'foo.y') """ % locals()) diff --git a/test/YACC/live.py b/test/YACC/live.py index a79d3db..4567dba 100644 --- a/test/YACC/live.py +++ b/test/YACC/live.py @@ -37,18 +37,15 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() -yacc = test.where_is('yacc') or test.where_is('bison') +yacc = test.where_is('yacc') or test.where_is('bison') or test.where_is('win_bison') if not yacc: test.skip_test('No yacc or bison found; skipping test.\n') -if sys.platform == 'win32': - if not test.where_is('gcc'): - test.skip_test('No gcc found on windows; skipping test.\n') - test.file_fixture('wrapper.py') test.write('SConstruct', """ +DefaultEnvironment(tools=[]) foo = Environment(YACCFLAGS='-d', tools = ['default', 'yacc']) yacc = foo.Dictionary('YACC') bar = Environment(YACC = r'%(_python_)s wrapper.py ' + yacc, tools = ['default', 'yacc']) -- cgit v0.12 From 9bbb976181f4cd5b6261993bae9fb5535bf07587 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 26 Feb 2019 21:52:46 -0800 Subject: On windows first try with native file paths with \\ then swap path to normalized path string with / separators. On a fresh windows build the node string will have windows dirsep and not normalizd. This yielded broken builds for the Meta project' --- src/engine/SCons/Node/FS.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 77c340f..f2ba7be 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3338,17 +3338,29 @@ class File(Base): # First try the simple name for node c_str = str(self) + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df + if not df: try: # this should yield a path which matches what's in the sconsign c_str = self.get_path() + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df except AttributeError as e: raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str) @@ -3388,16 +3400,23 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True - prev_ni = self._get_previous_signatures(dependency_map) + new_prev_ni = self._get_previous_signatures(dependency_map) + new = self.changed_timestamp_match(target, new_prev_ni) + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) + - if not self.changed_timestamp_match(target, prev_ni): + if not new: try: # NOTE: We're modifying the current node's csig in a query. - self.get_ninfo().csig = prev_ni.csig + self.get_ninfo().csig = new_prev_ni.csig except AttributeError: pass return False - return self.changed_content(target, prev_ni) + return self.changed_content(target, new_prev_ni) def changed_timestamp_newer(self, target, prev_ni): try: -- cgit v0.12 From 9c6608bc1069d8bfd199e30801b2d1118c0137af Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 1 Mar 2019 12:26:21 -0800 Subject: Added logic to shortcut comparing prev_ni if there is no dependency map from previous build. This should speed up md5-timestamp builds for clean builds. Also added debug logic to dump and check aagainst previous implementation at top of FS.PY MD5_TIMESTAMP_DEBUG flag. currently set to False --- src/engine/SCons/Node/FS.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f2ba7be..f520af1 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -61,6 +61,8 @@ from . import DeciderNeedsNode print_duplicate = 0 +MD5_TIMESTAMP_DEBUG = False + def sconsign_none(node): raise NotImplementedError @@ -3335,9 +3337,16 @@ class File(Base): List of csigs for provided list of children """ prev = [] + # MD5_TIMESTAMP_DEBUG = False + + if len(dmap) == 0: + if MD5_TIMESTAMP_DEBUG: print("Nothing dmap shortcutting") + return None + if MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap)) # First try the simple name for node c_str = str(self) + if MD5_TIMESTAMP_DEBUG: print("Checking :%s"%c_str) df = dmap.get(c_str, None) if df: return df @@ -3345,6 +3354,7 @@ class File(Base): if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3353,12 +3363,14 @@ class File(Base): # this should yield a path which matches what's in the sconsign c_str = self.get_path() df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3400,13 +3412,21 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True + if len(dependency_map) == 0: + # If there's no dependency map, there's no need to find the + # prev_ni as there aren't any + # shortcut the rest of the logic + if MD5_TIMESTAMP_DEBUG: print("Skipping checks len(dmap)=0") + return True new_prev_ni = self._get_previous_signatures(dependency_map) new = self.changed_timestamp_match(target, new_prev_ni) - old = self.changed_timestamp_match(target, prev_ni) - if old != new: - print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) - new_prev_ni = self._get_previous_signatures(dependency_map) + if MD5_TIMESTAMP_DEBUG: + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) if not new: -- cgit v0.12 From 3a107dcd6c93520f41c54b25a322317bfd1b8460 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 2 Mar 2019 21:56:53 -0500 Subject: Add test to cover this issue with MD5-timestamp decider breaking dependencies only on windows when one generated souce implicitly depends on another generated source. --- test/Decider/MD5-winonly-firstbuild.py | 60 +++++++++++++++++++++++++ test/Decider/MD5-winonly-fixture/SConstruct | 9 ++++ test/Decider/MD5-winonly-fixture/sconstest.skip | 0 test/Decider/MD5-winonly-fixture/test_lex.l | 10 +++++ test/Decider/MD5-winonly-fixture/test_parse.y | 43 ++++++++++++++++++ 5 files changed, 122 insertions(+) create mode 100644 test/Decider/MD5-winonly-firstbuild.py create mode 100644 test/Decider/MD5-winonly-fixture/SConstruct create mode 100644 test/Decider/MD5-winonly-fixture/sconstest.skip create mode 100644 test/Decider/MD5-winonly-fixture/test_lex.l create mode 100644 test/Decider/MD5-winonly-fixture/test_parse.y diff --git a/test/Decider/MD5-winonly-firstbuild.py b/test/Decider/MD5-winonly-firstbuild.py new file mode 100644 index 0000000..4047ea8 --- /dev/null +++ b/test/Decider/MD5-winonly-firstbuild.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test but which only shows on windows when one generated file depends on another generated file +In this case flex and yacc are an example. +""" + +import os +import stat + +import TestSCons +from TestCmd import IS_WINDOWS + + +test = TestSCons.TestSCons() + +if IS_WINDOWS: + yacc = test.where_is('yacc') or test.where_is('bison') or test.where_is('win_bison') + lex = test.where_is('flex') or test.where_is('lex') or test.where_is('win_flex') + # print("Lex:%s yacc:%s"%(lex,yacc)) + if not yacc or not lex: + test.skip("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test") +else: + test.skip("Windows only test") + + +test.dir_fixture('MD5-winonly-fixture') +test.run() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Decider/MD5-winonly-fixture/SConstruct b/test/Decider/MD5-winonly-fixture/SConstruct new file mode 100644 index 0000000..c194c47 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/SConstruct @@ -0,0 +1,9 @@ +DefaultEnvironment(tools=[]) +env=Environment() +env.Decider('MD5-timestamp') +env.AppendENVPath('PATH', r'd:\mesa\flexbison') +env.Tool('lex') +env.Tool('yacc') +env['YACCFLAGS'] = '-d' +env.Append(LEXFLAGS = ['--wincompat']) +env.SharedLibrary('dummy',['test_lex.l','test_parse.y']) diff --git a/test/Decider/MD5-winonly-fixture/sconstest.skip b/test/Decider/MD5-winonly-fixture/sconstest.skip new file mode 100644 index 0000000..e69de29 diff --git a/test/Decider/MD5-winonly-fixture/test_lex.l b/test/Decider/MD5-winonly-fixture/test_lex.l new file mode 100644 index 0000000..269b984 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/test_lex.l @@ -0,0 +1,10 @@ +%{ + +#include +#include "test_parse.h" +int c; +%} +%% + +%% + diff --git a/test/Decider/MD5-winonly-fixture/test_parse.y b/test/Decider/MD5-winonly-fixture/test_parse.y new file mode 100644 index 0000000..3ead9d2 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/test_parse.y @@ -0,0 +1,43 @@ +%{ +#include + +int regs[26]; +int base; + +%} + +%start digit + +%union { int a; } + + +%token DIGIT LETTER + +%left '|' +%left '&' +%left '+' '-' +%left '*' '/' '%' +%left UMINUS /*supplies precedence for unary minus */ + +%% /* beginning of rules section */ + +digit: DIGIT; + + +%% +main() +{ + return(yyparse()); +} + +yyerror(s) +char *s; +{ + fprintf(stderr, "%s\n",s); + return(0); +} + +yywrap() +{ + return(1); +} \ No newline at end of file -- cgit v0.12