From fb49becabc0cc641eef911473fdabc6c3b8e4788 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 25 Mar 2022 12:39:07 -0600 Subject: Add a bit more info to tool "generate" function [skip appveyor] Seemed useful to actually record the suggestion that tool modules should not unconditionally set the values of all contstruction variables - some could be considered user-settable and if so, existing values should be respected. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 47aba1f..e3466e2 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -7219,23 +7219,38 @@ A tool specification module must include two functions: generate(env, **kwargs) -Modifies the &consenv; env -to set up necessary &consvars; so that the facilities represented by -the tool can be executed. -It may use any keyword arguments -that the user supplies in kwargs +Modify the &consenv; env +to set up necessary &consvars;, Builders, Emitters, etc., +so the facilities represented by the tool can be executed. +Care should be taken not to overwrite &consvars; intended +to be settable by the user. For example: + + +def generate(env): + ... + if 'MYTOOL' not in env: + env['MYTOOL'] = env.Detect("mytool") + if 'MYTOOLFLAGS' not in env: + env['MYTOOLFLAGS'] = SCons.Util.CLVar('--myarg') + ... + + +The generate function +may use any keyword arguments +that the user supplies via kwargs to vary its initialization. exists(env) -Returns True if the tool can +Return a true value if the tool can be called in the context of env. +else false. Usually this means looking up one or more known programs using the PATH from the supplied env, but the tool can -make the "exists" decision in any way it chooses. +make the exists decision in any way it chooses. -- cgit v0.12 From 2b673e30e81f83304746e7fec9aa4ddb0fd66f88 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 22 Mar 2022 14:11:13 -0600 Subject: Fix some more tests to use context-mgr open Fiddled formatting a bit. Signed-off-by: Mats Wichmann --- test/Actions/append.py | 16 ++--- test/Actions/function.py | 64 +++++++++----------- test/Actions/pre-post.py | 31 +++++----- test/option/option--.py | 24 ++++---- test/option/option--Q.py | 27 ++++----- test/option/option-i.py | 51 ++++++++-------- test/option/option-k.py | 152 +++++++++++++++++++++++++---------------------- test/option/option-s.py | 23 ++++--- 8 files changed, 197 insertions(+), 191 deletions(-) diff --git a/test/Actions/append.py b/test/Actions/append.py index b5d4c3a..42a414b 100644 --- a/test/Actions/append.py +++ b/test/Actions/append.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,13 +22,13 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the addition operator of Action objects. -# Using Environment.Prepend() and Environment.Append(), you should be -# able to add new actions to existing ones, effectively adding steps -# to a build process. -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the addition operator of Action objects. +Using Environment.Prepend() and Environment.Append(), you should be +able to add new actions to existing ones, effectively adding steps +to a build process. +""" import os import stat diff --git a/test/Actions/function.py b/test/Actions/function.py index 7f292cf..699cd17 100644 --- a/test/Actions/function.py +++ b/test/Actions/function.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import sys @@ -59,6 +58,7 @@ options.AddVariables( ('literal_in_listcomp', 'Literal inside list comprehension', '2'), ) +DefaultEnvironment(tools=[]) optEnv = Environment(options=options, tools=[]) r = re.compile(optEnv['regexp']) @@ -67,25 +67,24 @@ withClosure = \ r''' def toto(header='%(header)s', trailer='%(trailer)s'): xxx = %(closure_cell_value)s - def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , - header=header, trailer=trailer): + def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , header=header, trailer=trailer): """+'"""%(docstring)s"""'+""" def foo(b=b): return %(nestedfuncexp)s - f = open(str(target[0]),'wb') - f.write(bytearray(header,'utf-8')) - for d in env['ENVDEPS']: - f.write(bytearray(d+'%(separator)s','utf-8')) - f.write(bytearray(trailer+'\\n','utf-8')) - f.write(bytearray(str(foo())+'\\n','utf-8')) - f.write(bytearray(r.match('aaaa').group(1)+'\\n','utf-8')) - f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1,2]]))+'\\n', 'utf-8')) - %(extracode)s - try: - f.write(bytearray(str(xarg),'utf-8')+b'\\n') - except NameError: - pass - f.close() + + with open(str(target[0]), 'wb') as f: + f.write(bytearray(header, 'utf-8')) + for d in env['ENVDEPS']: + f.write(bytearray(d+'%(separator)s', 'utf-8')) + f.write(bytearray(trailer+'\\n', 'utf-8')) + f.write(bytearray(str(foo())+'\\n', 'utf-8')) + f.write(bytearray(r.match('aaaa').group(1)+'\\n', 'utf-8')) + f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1, 2]]))+'\\n', 'utf-8')) + %(extracode)s + try: + f.write(bytearray(str(xarg), 'utf-8')+b'\\n') + except NameError: + pass return writeDeps ''' @@ -93,13 +92,9 @@ def toto(header='%(header)s', trailer='%(trailer)s'): exec(withClosure % optEnv) genHeaderBld = SCons.Builder.Builder( - action = SCons.Action.Action( - toto(), - 'Generating $TARGET', - varlist=['ENVDEPS'] - ), - suffix = '.gen.h' - ) + action=SCons.Action.Action(toto(), 'Generating $TARGET', varlist=['ENVDEPS']), + suffix='.gen.h', +) DefaultEnvironment(tools=[]) env = Environment(tools=[]) @@ -128,23 +123,22 @@ scons: done building targets. """ def runtest(arguments, expectedOutFile, expectedRebuild=True, stderr=""): - test.run(arguments=arguments, - stdout=expectedRebuild and rebuildstr or nobuildstr, - stderr="") + test.run( + arguments=arguments, + stdout=expectedRebuild and rebuildstr or nobuildstr, + stderr="", + ) sys.stdout.write('First Build.\n') test.must_match('Out.gen.h', expectedOutFile, message="First Build") - # Should not be rebuild when run a second time with the same - # arguments. - + # Should not be rebuild when run a second time with the same arguments. sys.stdout.write('Rebuild.\n') - test.run(arguments = arguments, stdout=nobuildstr, stderr="") + test.run(arguments=arguments, stdout=nobuildstr, stderr="") test.must_match('Out.gen.h', expectedOutFile, message="Should not rebuild") - # We're making this script chatty to prevent timeouts on really really # slow buildbot slaves (*cough* Solaris *cough*). diff --git a/test/Actions/pre-post.py b/test/Actions/pre-post.py index 358aa43..ce8cbdc 100644 --- a/test/Actions/pre-post.py +++ b/test/Actions/pre-post.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,12 +22,11 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the AddPreAction() and AddPostAction() API -# functions, which add pre-build and post-build actions to nodes. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the AddPreAction() and AddPostAction() API +functions, which add pre-build and post-build actions to nodes. +""" import os @@ -56,11 +57,8 @@ def before(env, target, source): def after(env, target, source): t = str(target[0]) a = "after_" + t - fin = open(t, "rb") - fout = open(a, "wb") - fout.write(fin.read()) - fout.close() - fin.close() + with open(t, "rb") as fin, open(a, "wb") as fout: + fout.write(fin.read()) os.chmod(a, os.stat(a)[stat.ST_MODE] | stat.S_IXUSR) foo = env.Program(source='foo.c', target='foo') @@ -103,19 +101,22 @@ test.must_match(['work3', 'dir', 'file'], "build()\n") # work4 start test.write(['work4', 'SConstruct'], """\ - DefaultEnvironment(tools=[]) def pre_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('pre %%s\\n' %% source[0]).encode()) + def post_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('post %%s\\n' %% source[0]).encode()) + env = Environment(tools=[]) -o = env.Command(['pre-post', 'file.out'], - 'file.in', - r'%(_python_)s build.py ${TARGETS[1]} $SOURCE') +o = env.Command( + ['pre-post', 'file.out'], + 'file.in', + r'%(_python_)s build.py ${TARGETS[1]} $SOURCE' +) env.AddPreAction(o, pre_action) env.AddPostAction(o, post_action) """ % locals()) diff --git a/test/option/option--.py b/test/option/option--.py index 8e06260..dbfb3c9 100644 --- a/test/option/option--.py +++ b/test/option/option--.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,16 +33,17 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ -MyBuild = Builder(action = r'%(_python_)s build.py $TARGETS') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = '-f1.out', source = 'f1.in') -env.MyBuild(target = '-f2.out', source = 'f2.in') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGETS') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='-f1.out', source='f1.in') +env.MyBuild(target='-f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") @@ -51,7 +51,7 @@ test.write('f2.in', "f2.in\n") expect = test.wrap_stdout('%(_python_)s build.py -f1.out\n%(_python_)s build.py -f2.out\n' % locals()) -test.run(arguments = '-- -f1.out -f2.out', stdout = expect) +test.run(arguments='-- -f1.out -f2.out', stdout=expect) test.fail_test(not os.path.exists(test.workpath('-f1.out'))) test.fail_test(not os.path.exists(test.workpath('-f2.out'))) diff --git a/test/option/option--Q.py b/test/option/option--Q.py index f3b82f9..ac56d34 100644 --- a/test/option/option--Q.py +++ b/test/option/option--Q.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -35,24 +34,24 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ - AddOption('--use_SetOption', action='store_true', dest='setoption', default=False) -use_setoption=GetOption('setoption') +use_setoption = GetOption('setoption') if use_setoption: SetOption('no_progress', True) - -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = 'f1.out', source = 'f1.in') -env.MyBuild(target = 'f2.out', source = 'f2.in') + +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='f1.out', source='f1.in') +env.MyBuild(target='f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") diff --git a/test/option/option-i.py b/test/option/option-i.py index 9b5212d..e426e1f 100644 --- a/test/option/option-i.py +++ b/test/option/option-i.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,43 +33,46 @@ test = TestSCons.TestSCons() test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) test.write('SConstruct', """ -Succeed = Builder(action = r'%(_python_)s succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s fail.py $TARGETS') -env = Environment(BUILDERS = { 'Succeed' : Succeed, 'Fail' : Fail }) -env.Fail(target = 'aaa.1', source = 'aaa.in') -env.Succeed(target = 'aaa.out', source = 'aaa.1') -env.Fail(target = 'bbb.1', source = 'bbb.in') -env.Succeed(target = 'bbb.out', source = 'bbb.1') +Succeed = Builder(action=r'%(_python_)s succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}) +env.Fail(target='aaa.1', source='aaa.in') +env.Succeed(target='aaa.out', source='aaa.1') +env.Fail(target='bbb.1', source='bbb.in') +env.Succeed(target='bbb.out', source='bbb.1') """ % locals()) test.write('aaa.in', "aaa.in\n") test.write('bbb.in', "bbb.in\n") -test.run(arguments = 'aaa.1 aaa.out bbb.1 bbb.out', - stderr = 'scons: *** [aaa.1] Error 1\n', - status = 2) +test.run( + arguments='aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(os.path.exists(test.workpath('aaa.out'))) test.fail_test(os.path.exists(test.workpath('bbb.1'))) test.fail_test(os.path.exists(test.workpath('bbb.out'))) -test.run(arguments = '-i aaa.1 aaa.out bbb.1 bbb.out', - stderr = - 'scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='-i aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out',mode='r') != "succeed.py: aaa.out\n") @@ -80,9 +82,10 @@ test.fail_test(test.read('bbb.out',mode='r') != "succeed.py: bbb.out\n") test.unlink("aaa.out") test.unlink("bbb.out") -test.run(arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', - stderr='scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out', mode='r') != "succeed.py: aaa.out\n") diff --git a/test/option/option-k.py b/test/option/option-k.py index d6c81ea..6a7cfcb 100644 --- a/test/option/option-k.py +++ b/test/option/option-k.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -33,18 +32,17 @@ test = TestSCons.TestSCons() test.subdir('work1', 'work2', 'work3') - - test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) @@ -66,19 +64,23 @@ env.Succeed(target='bbb.out', source='bbb.in') test.write(['work1', 'aaa.in'], "aaa.in\n") test.write(['work1', 'bbb.in'], "bbb.in\n") -test.run(chdir='work1', - arguments='aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) test.must_not_exist(test.workpath('work1', 'bbb.out')) -test.run(chdir='work1', - arguments='-k aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='-k aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -86,10 +88,12 @@ test.must_match(['work1', 'bbb.out'], "succeed.py: bbb.out\n", mode='r') test.unlink(['work1', 'bbb.out']) -test.run(chdir = 'work1', - arguments='--keep-going aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='--keep-going aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -131,11 +135,12 @@ env.Succeed('ddd.out', 'ccc.in') test.write(['work2', 'aaa.in'], "aaa.in\n") test.write(['work2', 'ccc.in'], "ccc.in\n") -test.run(chdir='work2', - arguments='-k .', - status=2, - stderr=None, - stdout="""\ +test.run( + chdir='work2', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -143,7 +148,9 @@ scons: Building targets ... %(_python_)s ../succeed.py ccc.out %(_python_)s ../succeed.py ddd.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work2', 'aaa.out']) test.must_not_exist(['work2', 'bbb.out']) @@ -175,19 +182,19 @@ test.must_match(['work2', 'ddd.out'], "succeed.py: ddd.out\n", mode='r') test.write(['work3', 'SConstruct'], """\ DefaultEnvironment(tools=[]) -Succeed = Builder(action = r'%(_python_)s ../succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s ../fail.py $TARGETS') -env = Environment(BUILDERS = {'Succeed': Succeed, 'Fail': Fail}, tools=[]) +Succeed = Builder(action=r'%(_python_)s ../succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s ../fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}, tools=[]) a = env.Fail('aaa.out', 'aaa.in') b = env.Succeed('bbb.out', 'bbb.in') c = env.Succeed('ccc.out', 'ccc.in') -a1 = Alias( 'a1', a ) -a2 = Alias( 'a2', a+b) -a4 = Alias( 'a4', c) -a3 = Alias( 'a3', a4+c) +a1 = Alias('a1', a) +a2 = Alias('a2', a + b) +a4 = Alias('a4', c) +a3 = Alias('a3', a4 + c) -Alias('all', a1+a2+a3) +Alias('all', a1 + a2 + a3) """ % locals()) test.write(['work3', 'aaa.in'], "aaa.in\n") @@ -196,36 +203,37 @@ test.write(['work3', 'ccc.in'], "ccc.in\n") # Test tegular build (i.e. without -k) -test.run(chdir = 'work3', - arguments = '.', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='.', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... %(_python_)s ../fail.py aaa.out scons: building terminated because of errors. -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Current directory -test.run(chdir = 'work3', - arguments = '-k .', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -233,26 +241,27 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) # Single target -test.run(chdir = 'work3', - arguments = '--keep-going all', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='--keep-going all', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -260,26 +269,26 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Separate top-level targets -test.run(chdir = 'work3', - arguments = '-k a1 a2 a3', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k a1 a2 a3', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -287,13 +296,14 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - test.pass_test() # Local Variables: diff --git a/test/option/option-s.py b/test/option/option-s.py index 89a0c62..359c295 100644 --- a/test/option/option-s.py +++ b/test/option/option-s.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,18 +33,19 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ DefaultEnvironment(tools=[]) -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') -silent = ARGUMENTS.get('QUIET',0) +silent = ARGUMENTS.get('QUIET', 0) if silent: - SetOption('silent',True) + SetOption('silent', True) env = Environment(BUILDERS={'MyBuild': MyBuild}, tools=[]) env.MyBuild(target='f1.out', source='f1.in') @@ -86,10 +86,7 @@ test.run(arguments='QUIET=1 f1.out f2.out', test.fail_test(not os.path.exists(test.workpath('f1.out'))) test.fail_test(not os.path.exists(test.workpath('f2.out'))) - - test.pass_test() - # Local Variables: # tab-width:4 -- cgit v0.12 From c0469d3b29141e49ff00b7ddcb5aba5660c37b8f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 3 Apr 2022 10:52:42 -0600 Subject: Fix a ref-before-assign issue in tex tool Also some formatting fiddles. Fixes #2888 Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + SCons/Tool/tex.py | 52 +++++++++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9e77793..c1fa7d2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -72,6 +72,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER or the super() two-argument syntax. - Renamed ParseFlag's internal data structure to "mapping" instead of "dict" (avoid redefining builtin) + - Fix an old use-before-set bug in tex tool (issue #2888) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index d8b694e..6d0a5fb 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -492,21 +492,23 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None return result -def LaTeXAuxAction(target = None, source= None, env=None): - result = InternalLaTeXAuxAction( LaTeXAction, target, source, env ) +def LaTeXAuxAction(target=None, source=None, env=None): + result = InternalLaTeXAuxAction(LaTeXAction, target, source, env) return result + LaTeX_re = re.compile("\\\\document(style|class)") -def is_LaTeX(flist,env,abspath): + +def is_LaTeX(flist, env, abspath) -> bool: """Scan a file list to decide if it's TeX- or LaTeX-flavored.""" # We need to scan files that are included in case the # \documentclass command is in them. # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS'] - savedpath = modify_env_var(env, 'TEXINPUTS', abspath) - paths = env['ENV']['TEXINPUTS'] + savedpath = modify_env_var(env, "TEXINPUTS", abspath) + paths = env["ENV"]["TEXINPUTS"] if SCons.Util.is_List(paths): pass else: @@ -516,54 +518,58 @@ def is_LaTeX(flist,env,abspath): # now that we have the path list restore the env if savedpath is _null: try: - del env['ENV']['TEXINPUTS'] + del env["ENV"]["TEXINPUTS"] except KeyError: - pass # was never set + pass # was never set else: - env['ENV']['TEXINPUTS'] = savedpath + env["ENV"]["TEXINPUTS"] = savedpath if Verbose: - print("is_LaTeX search path ",paths) - print("files to search :",flist) + print("is_LaTeX search path ", paths) + print("files to search: ", flist) # Now that we have the search path and file list, check each one + file_test = False for f in flist: if Verbose: - print(" checking for Latex source ",str(f)) + print(f" checking for Latex source {f}") content = f.get_text_contents() if LaTeX_re.search(content): if Verbose: - print("file %s is a LaTeX file" % str(f)) - return 1 + print(f"file {f} is a LaTeX file") + return True if Verbose: - print("file %s is not a LaTeX file" % str(f)) + print(f"file {f} is not a LaTeX file") # now find included files - inc_files = [ ] - inc_files.extend( include_re.findall(content) ) + inc_files = [] + inc_files.extend(include_re.findall(content)) if Verbose: - print("files included by '%s': "%str(f),inc_files) + print(f"files included by '{f}': ", inc_files) # inc_files is list of file names as given. need to find them # using TEXINPUTS paths. # search the included files for src in inc_files: - srcNode = FindFile(src,['.tex','.ltx','.latex'],paths,env,requireExt=False) + srcNode = FindFile( + src, [".tex", ".ltx", ".latex"], paths, env, requireExt=False + ) # make this a list since is_LaTeX takes a list. - fileList = [srcNode,] + fileList = [srcNode] if Verbose: - print("FindFile found ",srcNode) + print("FindFile found ", srcNode) if srcNode is not None: file_test = is_LaTeX(fileList, env, abspath) # return on first file that finds latex is needed. if file_test: - return file_test + return True if Verbose: - print(" done scanning ",str(f)) + print(f" done scanning {f}") + + return False - return 0 def TeXLaTeXFunction(target = None, source= None, env=None): """A builder for TeX and LaTeX that scans the source file to -- cgit v0.12 From f0f5657f3214bf50f56cf076c9ecbe37c1fd0d75 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 5 Apr 2022 12:29:32 -0600 Subject: test framework: fix exception on timeout If the framework wait_for() method actually times out, it tries to return stdout and stderr by calling the framework methods of those names. The stdout() method was protected against the message not having been captured (as is the case on timeout). Updated the stderr() method to use the same technique. Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + testing/framework/TestCmd.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c1fa7d2..8390f00 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -73,6 +73,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Renamed ParseFlag's internal data structure to "mapping" instead of "dict" (avoid redefining builtin) - Fix an old use-before-set bug in tex tool (issue #2888) + - Fix a test harness exception returning stderr if a wait_for timed out. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 79b16f5..5759121 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -311,6 +311,7 @@ import time import traceback from collections import UserList, UserString from subprocess import PIPE, STDOUT +from typing import Optional IS_WINDOWS = sys.platform == 'win32' IS_MACOS = sys.platform == 'darwin' @@ -1641,7 +1642,7 @@ class TestCmd: """ time.sleep(seconds) - def stderr(self, run=None): + def stderr(self, run=None) -> Optional[str]: """Returns the error output from the specified run number. If there is no specified run number, then returns the error @@ -1653,10 +1654,13 @@ class TestCmd: run = len(self._stderr) elif run < 0: run = len(self._stderr) + run - run = run - 1 - return self._stderr[run] + run -= 1 + try: + return self._stderr[run] + except IndexError: + return None - def stdout(self, run=None): + def stdout(self, run=None) -> Optional[str]: """Returns the stored standard output from a given run. Args: @@ -1673,7 +1677,7 @@ class TestCmd: run = len(self._stdout) elif run < 0: run = len(self._stdout) + run - run = run - 1 + run -= 1 try: return self._stdout[run] except IndexError: -- cgit v0.12 From f842abcfaeeb8a4ce911a2b0d076ec1ae929c8cd Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 6 Apr 2022 17:25:28 -0400 Subject: Refactor handling of host/target combinations: replace existing data structures containing pseudo-targets with defined and derived data structures the contain lists of host/target combinations that implicitly handle differences between regular versions and express versions of msvc. Refactor the method for returning host/target combination candidates and check for known unsupported combinations. Remove the now redundant function to check the host/target combination is supported for 7.1 and earlier. Expand the error detection when processing script out by adding known prefixes of error messages using a regular expression. Expand the host/target search list during detection to properly pick up express versions with 64-bit targets. --- SCons/Tool/MSCommon/vc.py | 603 ++++++++++++++++++++++++----------------- SCons/Tool/MSCommon/vcTests.py | 85 ++++-- 2 files changed, 418 insertions(+), 270 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index fe31cb3..d9160a9 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -45,6 +45,7 @@ import os import platform from string import digits as string_digits from subprocess import PIPE +import re import SCons.Util import SCons.Warnings @@ -96,84 +97,202 @@ _ARCH_TO_CANONICAL = { "aarch64" : "arm64", } -# Starting with 14.1 (aka VS2017), the tools are organized by host directory. -# subdirs for each target. They are now in .../VC/Auxuiliary/Build. -# Note 2017 Express uses Hostx86 even if it's on 64-bit Windows, -# not reflected in this table. -_HOST_TARGET_TO_CL_DIR_GREATER_THAN_14 = { - ("amd64","amd64") : ("Hostx64","x64"), - ("amd64","x86") : ("Hostx64","x86"), - ("amd64","arm") : ("Hostx64","arm"), - ("amd64","arm64") : ("Hostx64","arm64"), - ("x86","amd64") : ("Hostx86","x64"), - ("x86","x86") : ("Hostx86","x86"), - ("x86","arm") : ("Hostx86","arm"), - ("x86","arm64") : ("Hostx86","arm64"), +# utility methods for derived data structure construction + +def _make_host_target_map(all_hosts, all_targets): + # host_target_map[host][target] -> host_target_list + host_target_map = {} + for host, host_list in all_hosts.items(): + host_target_map[host] = {} + for host_platform in host_list: + for target_platform in all_targets[host_platform]: + if target_platform not in host_target_map[host]: + host_target_map[host][target_platform] = [] + host_target_map[host][target_platform].append((host_platform, target_platform)) + return host_target_map + +def _make_host_all_targets_map(host_target_map, all_targets): + # host_all_target_map[host] -> host_target_list + # special host key '_all_' contains all (host,target) combinations + all = '_all_' + host_all_target_map = {} + host_all_target_map[all] = [] + for host in host_target_map.keys(): + host_all_target_map[host] = [] + for target in all_targets[host]: + for host_target in host_target_map[host][target]: + for key in (host, all): + if host_target not in host_all_target_map[key]: + host_all_target_map[key].append(host_target) + return host_all_target_map + +def _make_host_default_target_map(host_target_map, default_targets): + # host_default_target_map[host] -> host_target_list + host_default_target_map = {} + for host in host_target_map.keys(): + host_default_target_map[host] = [] + for target in default_targets[host]: + host_default_target_map[host].extend(host_target_map[host][target]) + return host_default_target_map + +def _make_target_host_map(host_target_map): + # target_host_map[target][host] -> host_target_list + target_host_map = {} + for host in host_target_map.keys(): + for target in host_target_map[host].keys(): + if target not in target_host_map: + target_host_map[target] = {} + if host not in target_host_map[target]: + target_host_map[target][host] = [] + for host_target in host_target_map[host][target]: + if host_target not in target_host_map[target][host]: + target_host_map[target][host].append(host_target) + return target_host_map + +# 14.1 (VS2017) and later + +# Given a (host, target) tuple, return a tuple containing the batch file to +# look for and the path fragment to find cl.exe. We can't rely on returning +# an arg to use for vcvarsall.bat, because that script will run even if given +# a host/target pair that isn't installed. +# +# Starting with 14.1 (VS2017), the batch files are located in directory +# /VC/Auxiliary/Build. The batch file name is the first value of the +# stored tuple. +# +# The build tools are organized by host and target subdirectories under each toolset +# version directory. For example, /VC/Tools/MSVC/14.31.31103/bin/Hostx64/x64. +# The cl path fragment under the toolset version folder is the second value of +# the stored tuple. +# +# Note: 2017 Express uses Hostx86 even if it's on 64-bit Windows which is not +# reflected in this table. + +_GE2017_HOST_TARGET_BATCHFILE_CLPATH = { + + ('amd64', 'amd64') : ( 'vcvars64.bat', r'bin\Hostx64\x64' ), + ('amd64', 'x86') : ( 'vcvarsamd64_x86.bat', r'bin\Hostx64\x86' ), + ('amd64', 'arm') : ( 'vcvarsamd64_arm.bat', r'bin\Hostx64\arm' ), + ('amd64', 'arm64') : ( 'vcvarsamd64_arm64.bat', r'bin\Hostx64\arm64' ), + + ('x86', 'amd64') : ( 'vcvarsx86_amd64.bat', r'bin\Hostx86\x64' ), + ('x86', 'x86') : ( 'vcvars32.bat', r'bin\Hostx86\x86' ), + ('x86', 'arm') : ( 'vcvarsx86_arm.bat', r'bin\Hostx86\arm' ), + ('x86', 'arm64') : ( 'vcvarsx86_arm64.bat', r'bin\Hostx86\arm64' ), + +} + +_GE2017_HOST_ALL_HOSTS = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], +} + +_GE2017_HOST_ALL_TARGETS = { + 'amd64': ['amd64', 'x86', 'arm', 'arm64'], + 'x86': ['amd64', 'x86', 'arm', 'arm64'], +} + +_GE2017_HOST_DEFAULT_TARGETS = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], +} + +_GE2017_HOST_TARGET_MAP = _make_host_target_map(_GE2017_HOST_ALL_HOSTS, _GE2017_HOST_ALL_TARGETS) +_GE2017_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_GE2017_HOST_TARGET_MAP, _GE2017_HOST_ALL_TARGETS) +_GE2017_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_GE2017_HOST_TARGET_MAP, _GE2017_HOST_DEFAULT_TARGETS) +_GE2017_TARGET_HOST_MAP = _make_target_host_map(_GE2017_HOST_TARGET_MAP) + +#debug("_GE2017_HOST_TARGET_MAP: %s", _GE2017_HOST_TARGET_MAP) +#debug("_GE2017_HOST_ALL_TARGETS_MAP: %s", _GE2017_HOST_ALL_TARGETS_MAP) +#debug("_GE2017_HOST_DEF_TARGETS_MAP: %s", _GE2017_HOST_DEF_TARGETS_MAP) +#debug("_GE2017_TARGET_HOST_MAP: %s", _GE2017_TARGET_HOST_MAP) + +# 14.0 (VS2015) to 8.0 (VS2005) + +# Given a (host, target) tuple, return a tuple containing the argument for +# the batch file and the the path fragment to find cl.exe. +# +# In 14.0 (VS2015) and earlier, the original x86 tools are in the tools +# bin directory (i.e., /VC/bin). Any other tools are in subdirectory +# named for the the host/target pair or a single name if the host==target. + +_LE2015_HOST_TARGET_BATCHARG_CLPATH = { + + ('amd64', 'amd64') : ( 'amd64', r'bin\amd64' ), + ('amd64', 'x86') : ( 'amd64_x86', r'bin\amd64_x86' ), + ('amd64', 'arm') : ( 'amd64_arm', r'bin\amd64_arm' ), + + ('x86', 'amd64') : ( 'x86_amd64', r'bin\x86_amd64' ), + ('x86', 'x86') : ( 'x86', r'bin' ), + ('x86', 'arm') : ( 'x86_arm', r'bin\x86_arm' ), + ('x86', 'ia64') : ( 'x86_ia64', r'bin\x86_ia64' ), + + ('arm', 'arm') : ( 'arm', r'bin\arm' ), + ('ia64', 'ia64') : ( 'ia64', r'bin\ia64' ), + +} + +_LE2015_HOST_ALL_HOSTS = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm': ['arm'], + 'ia64': ['ia64'], +} + +_LE2015_HOST_ALL_TARGETS = { + 'amd64': ['amd64', 'x86', 'arm'], + 'x86': ['amd64', 'x86', 'arm', 'ia64'], + 'arm': ['arm'], + 'ia64': ['ia64'], } -# before 14.1 (VS2017): the original x86 tools are in the tools dir, -# any others are in a subdir named by the host/target pair, -# or just a single word if host==target -_HOST_TARGET_TO_CL_DIR = { - ("amd64","amd64") : "amd64", - ("amd64","x86") : "amd64_x86", - ("amd64","arm") : "amd64_arm", - ("amd64","arm64") : "amd64_arm64", - ("x86","amd64") : "x86_amd64", - ("x86","x86") : "", - ("x86","arm") : "x86_arm", - ("x86","arm64") : "x86_arm64", - ("arm","arm") : "arm", +_LE2015_HOST_DEFAULT_TARGETS = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm': ['arm'], + 'ia64': ['ia64'], } -# 14.1 (VS2017) and later: -# Given a (host, target) tuple, return the batch file to look for. -# We can't rely on returning an arg to use for vcvarsall.bat, -# because that script will run even if given a pair that isn't installed. -# Targets that already look like a pair are pseudo targets that -# effectively mean to skip whatever the host was specified as. -_HOST_TARGET_TO_BAT_ARCH_GT14 = { - ("amd64", "amd64"): "vcvars64.bat", - ("amd64", "x86"): "vcvarsamd64_x86.bat", - ("amd64", "x86_amd64"): "vcvarsx86_amd64.bat", - ("amd64", "x86_x86"): "vcvars32.bat", - ("amd64", "arm"): "vcvarsamd64_arm.bat", - ("amd64", "x86_arm"): "vcvarsx86_arm.bat", - ("amd64", "arm64"): "vcvarsamd64_arm64.bat", - ("amd64", "x86_arm64"): "vcvarsx86_arm64.bat", - ("x86", "x86"): "vcvars32.bat", - ("x86", "amd64"): "vcvarsx86_amd64.bat", - ("x86", "x86_amd64"): "vcvarsx86_amd64.bat", - ("x86", "arm"): "vcvarsx86_arm.bat", - ("x86", "x86_arm"): "vcvarsx86_arm.bat", - ("x86", "arm64"): "vcvarsx86_arm64.bat", - ("x86", "x86_arm64"): "vcvarsx86_arm64.bat", +_LE2015_HOST_TARGET_MAP = _make_host_target_map(_LE2015_HOST_ALL_HOSTS, _LE2015_HOST_ALL_TARGETS) +_LE2015_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_LE2015_HOST_TARGET_MAP, _LE2015_HOST_ALL_TARGETS) +_LE2015_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_LE2015_HOST_TARGET_MAP, _LE2015_HOST_DEFAULT_TARGETS) +_LE2015_TARGET_HOST_MAP = _make_target_host_map(_LE2015_HOST_TARGET_MAP) + +#debug("_LE2015_HOST_TARGET_MAP: %s", _LE2015_HOST_TARGET_MAP) +#debug("_LE2015_HOST_ALL_TARGETS_MAP: %s", _LE2015_HOST_ALL_TARGETS_MAP) +#debug("_LE2015_HOST_DEF_TARGETS_MAP: %s", _LE2015_HOST_DEF_TARGETS_MAP) +#debug("_LE2015_TARGET_HOST_MAP: %s", _LE2015_TARGET_HOST_MAP) + +# 7.1 (VS2003) and earlier + +# For 7.1 (VS2003) and earlier, there are only x86 targets and the batch files +# take no arguments. + +_LE2003_HOST_ALL_HOSTS = { + 'amd64': ['x86'], + 'x86': ['x86'], } -# before 14.1 (VS2017): -# Given a (host, target) tuple, return the argument for the bat file; -# Both host and target should be canoncalized. -# If the target already looks like a pair, return it - these are -# pseudo targets (mainly used by Express versions) -_HOST_TARGET_ARCH_TO_BAT_ARCH = { - ("x86", "x86"): "x86", - ("x86", "amd64"): "x86_amd64", - ("x86", "x86_amd64"): "x86_amd64", - ("amd64", "x86_amd64"): "x86_amd64", # This is present in (at least) VS2012 express - ("amd64", "amd64"): "amd64", - ("amd64", "x86"): "x86", - ("amd64", "x86_x86"): "x86", - ("x86", "ia64"): "x86_ia64", # gone since 14.0 - ("x86", "arm"): "x86_arm", # since 14.0 - ("x86", "arm64"): "x86_arm64", # since 14.1 - ("amd64", "arm"): "amd64_arm", # since 14.0 - ("amd64", "arm64"): "amd64_arm64", # since 14.1 - ("x86", "x86_arm"): "x86_arm", # since 14.0 - ("x86", "x86_arm64"): "x86_arm64", # since 14.1 - ("amd64", "x86_arm"): "x86_arm", # since 14.0 - ("amd64", "x86_arm64"): "x86_arm64", # since 14.1 +_LE2003_HOST_ALL_TARGETS = { + 'amd64': ['x86'], + 'x86': ['x86'], } +_LE2003_HOST_DEFAULT_TARGETS = { + 'amd64': ['x86'], + 'x86': ['x86'], +} + +_LE2003_HOST_TARGET_MAP = _make_host_target_map(_LE2003_HOST_ALL_HOSTS, _LE2003_HOST_ALL_TARGETS) +_LE2003_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_LE2003_HOST_TARGET_MAP, _LE2003_HOST_ALL_TARGETS) +_LE2003_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_LE2003_HOST_TARGET_MAP, _LE2003_HOST_DEFAULT_TARGETS) +_LE2003_TARGET_HOST_MAP = _make_target_host_map(_LE2003_HOST_TARGET_MAP) + +#debug("_LE2003_HOST_TARGET_MAP: %s", _LE2003_HOST_TARGET_MAP) +#debug("_LE2003_HOST_ALL_TARGETS_MAP: %s", _LE2003_HOST_ALL_TARGETS_MAP) +#debug("_LE2003_HOST_DEF_TARGETS_MAP: %s", _LE2003_HOST_DEF_TARGETS_MAP) +#debug("_LE2003_TARGET_HOST_MAP: %s", _LE2003_TARGET_HOST_MAP) + _CL_EXE_NAME = 'cl.exe' def get_msvc_version_numeric(msvc_version): @@ -192,44 +311,104 @@ def get_msvc_version_numeric(msvc_version): """ return ''.join([x for x in msvc_version if x in string_digits + '.']) -def get_host_target(env): - host_platform = env.get('HOST_ARCH') - debug("HOST_ARCH:%s", str(host_platform)) - if not host_platform: - host_platform = platform.machine() +def get_host_platform(host_platform): + + host_platform = host_platform.lower() # Solaris returns i86pc for both 32 and 64 bit architectures - if host_platform == "i86pc": + if host_platform == 'i86pc': if platform.architecture()[0] == "64bit": host_platform = "amd64" else: host_platform = "x86" - # Retain user requested TARGET_ARCH - req_target_platform = env.get('TARGET_ARCH') - debug("TARGET_ARCH:%s", str(req_target_platform)) - if req_target_platform: - # If user requested a specific platform then only try that one. - target_platform = req_target_platform - else: - target_platform = host_platform - try: - host = _ARCH_TO_CANONICAL[host_platform.lower()] + host =_ARCH_TO_CANONICAL[host_platform] except KeyError: msg = "Unrecognized host architecture %s" raise MSVCUnsupportedHostArch(msg % repr(host_platform)) from None - try: - target = _ARCH_TO_CANONICAL[target_platform.lower()] - except KeyError: - all_archs = str(list(_ARCH_TO_CANONICAL.keys())) - raise MSVCUnsupportedTargetArch( - "Unrecognized target architecture %s\n\tValid architectures: %s" - % (target_platform, all_archs) - ) from None + return host + +_native_host_platform = None + +def get_native_host_platform(): + global _native_host_platform + + if _native_host_platform is None: + + _native_host_platform = get_host_platform(platform.machine()) + + return _native_host_platform + +def get_host_target(env, msvc_version, all_host_targets=False): + + vernum = float(get_msvc_version_numeric(msvc_version)) + + if vernum > 14: + # 14.1 (VS2017) and later + target_host_map = _GE2017_TARGET_HOST_MAP + host_all_targets_map = _GE2017_HOST_ALL_TARGETS_MAP + host_def_targets_map = _GE2017_HOST_DEF_TARGETS_MAP + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) + target_host_map = _LE2015_TARGET_HOST_MAP + host_all_targets_map = _LE2015_HOST_ALL_TARGETS_MAP + host_def_targets_map = _LE2003_HOST_DEF_TARGETS_MAP + else: + # 7.1 (VS2003) and earlier + target_host_map = _LE2003_TARGET_HOST_MAP + host_all_targets_map = _LE2015_HOST_ALL_TARGETS_MAP + host_def_targets_map = _LE2003_HOST_DEF_TARGETS_MAP - return (host, target, req_target_platform) + host_arch = env.get('HOST_ARCH') if env else None + debug("HOST_ARCH:%s", str(host_arch)) + + if host_arch: + host_platform = get_host_platform(host_arch) + else: + host_platform = get_native_host_platform() + + target_arch = env.get('TARGET_ARCH') if env else None + debug("TARGET_ARCH:%s", str(target_arch)) + + if target_arch: + + try: + target_platform = _ARCH_TO_CANONICAL[target_arch.lower()] + except KeyError: + all_archs = str(list(_ARCH_TO_CANONICAL.keys())) + raise MSVCUnsupportedTargetArch( + "Unrecognized target architecture %s\n\tValid architectures: %s" + % (repr(target_arch), all_archs) + ) from None + + try: + host_target_list = target_host_map[target_platform][host_platform] + except KeyError: + host_target_list = [] + warn_msg = "unsupported host, target combination ({}, {}) for MSVC version {}".format( + repr(host_platform), repr(target_platform), msvc_version + ) + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + debug(warn_msg) + + else: + + target_platform = None + + if all_host_targets: + host_targets_map = host_all_targets_map + else: + host_targets_map = host_def_targets_map + + try: + host_target_list = host_targets_map[host_platform] + except KeyError: + msg = "Unrecognized host architecture %s for version %s" + raise MSVCUnsupportedHostArch(msg % (repr(host_platform), msvc_version)) from None + + return (host_platform, target_platform, host_target_list) # If you update this, update SupportedVSList in Tool/MSCommon/vs.py, and the # MSVC_VERSION documentation in Tool/msvc.xml. @@ -336,29 +515,6 @@ def msvc_version_to_maj_min(msvc_version): raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) from None -def is_host_target_supported(host_target, msvc_version): - """Check if (host, target) pair is supported for a VC version. - - Only checks whether a given version *may* support the given - (host, target) pair, not that the toolchain is actually on the machine. - - Args: - host_target: canonalized host-target pair, e.g. - ("x86", "amd64") for cross compilation from 32- to 64-bit Windows. - msvc_version: Visual C++ version (major.minor), e.g. "10.0" - - Returns: - True or False - - """ - # We assume that any Visual Studio version supports x86 as a target - if host_target[1] != "x86": - maj, min = msvc_version_to_maj_min(msvc_version) - if maj < 8: - return False - return True - - VSWHERE_PATHS = [os.path.join(p,'vswhere.exe') for p in [ os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer"), os.path.expandvars(r"%ProgramFiles%\Microsoft Visual Studio\Installer"), @@ -493,15 +649,14 @@ def find_vc_pdir(env, msvc_version): raise MissingConfiguration("registry dir {} not found on the filesystem".format(comps)) return None -def find_batch_file(env,msvc_version,host_arch,target_arch): +def find_batch_file(env, msvc_version, host_arch, target_arch): """ Find the location of the batch script which should set up the compiler for any TARGET_ARCH whose compilers were installed by Visual Studio/VCExpress In newer (2017+) compilers, make use of the fact there are vcvars scripts named with a host_target pair that calls vcvarsall.bat properly, - so use that and return an indication we don't need the argument - we would have computed to run vcvarsall.bat. + so use that and return an empty argument. """ pdir = find_vc_pdir(env, msvc_version) if pdir is None: @@ -510,21 +665,26 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): # filter out e.g. "Exp" from the version name msvc_ver_numeric = get_msvc_version_numeric(msvc_version) - use_arg = True vernum = float(msvc_ver_numeric) - if 7 <= vernum < 8: + + arg = '' + if vernum > 14: + # 14.1 (VS2017) and later + batfiledir = os.path.join(pdir, "Auxiliary", "Build") + batfile, _ = _GE2017_HOST_TARGET_BATCHFILE_CLPATH[(host_arch, target_arch)] + batfilename = os.path.join(batfiledir, batfile) + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) + arg, _ = _LE2015_HOST_TARGET_BATCHARG_CLPATH[(host_arch, target_arch)] + batfilename = os.path.join(pdir, "vcvarsall.bat") + elif 8 > vernum >= 7: + # 7.1 (VS2003) to 7.0 (VS2003) pdir = os.path.join(pdir, os.pardir, "Common7", "Tools") batfilename = os.path.join(pdir, "vsvars32.bat") - elif vernum < 7: + else: + # 6.0 (VS6) and earlier pdir = os.path.join(pdir, "Bin") batfilename = os.path.join(pdir, "vcvars32.bat") - elif 8 <= vernum <= 14: - batfilename = os.path.join(pdir, "vcvarsall.bat") - else: # vernum >= 14.1 VS2017 and above - batfiledir = os.path.join(pdir, "Auxiliary", "Build") - targ = _HOST_TARGET_TO_BAT_ARCH_GT14[(host_arch, target_arch)] - batfilename = os.path.join(batfiledir, targ) - use_arg = False if not os.path.exists(batfilename): debug("Not found: %s", batfilename) @@ -532,16 +692,16 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): installed_sdks = get_installed_sdks() for _sdk in installed_sdks: - sdk_bat_file = _sdk.get_sdk_vc_script(host_arch,target_arch) + sdk_bat_file = _sdk.get_sdk_vc_script(host_arch, target_arch) if not sdk_bat_file: debug("batch file not found:%s", _sdk) else: - sdk_bat_file_path = os.path.join(pdir,sdk_bat_file) + sdk_bat_file_path = os.path.join(pdir, sdk_bat_file) if os.path.exists(sdk_bat_file_path): debug('sdk_bat_file_path:%s', sdk_bat_file_path) - return (batfilename, use_arg, sdk_bat_file_path) - return (batfilename, use_arg, None) + return (batfilename, arg, sdk_bat_file_path) + return (batfilename, arg, None) __INSTALLED_VCS_RUN = None _VC_TOOLS_VERSION_FILE_PATH = ['Auxiliary', 'Build', 'Microsoft.VCToolsVersion.default.txt'] @@ -552,8 +712,8 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): Locates cl in the vc_dir depending on TARGET_ARCH, HOST_ARCH and the msvc version. TARGET_ARCH and HOST_ARCH can be extracted from the - passed env, unless it is None, in which case the native platform is - assumed for both host and target. + passed env, unless the env is None, in which case the native platform is + assumed for the host and all associated targets. Args: env: Environment @@ -570,24 +730,16 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): """ - # determine if there is a specific target platform we want to build for and - # use that to find a list of valid VCs, default is host platform == target platform - # and same for if no env is specified to extract target platform from - if env: - (host_platform, target_platform, req_target_platform) = get_host_target(env) - else: - host_platform = platform.machine().lower() - target_platform = host_platform - - host_platform = _ARCH_TO_CANONICAL[host_platform] - target_platform = _ARCH_TO_CANONICAL[target_platform] + # Find the host, target, and all candidate (host, target) platform combinations: + platforms = get_host_target(env, msvc_version, all_host_targets=True) + debug("host_platform %s, target_platform %s host_target_list %s", *platforms) + host_platform, target_platform, host_target_list = platforms - debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - - ver_num = float(get_msvc_version_numeric(msvc_version)) + vernum = float(get_msvc_version_numeric(msvc_version)) # make sure the cl.exe exists meaning the tool is installed - if ver_num > 14: + if vernum > 14: + # 14.1 (VS2017) and later # 2017 and newer allowed multiple versions of the VC toolset to be # installed at the same time. This changes the layout. # Just get the default tool version for now @@ -603,63 +755,46 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('failed to find MSVC version in %s', default_toolset_file) return False - host_trgt_dir = _HOST_TARGET_TO_CL_DIR_GREATER_THAN_14.get((host_platform, target_platform), None) - if host_trgt_dir is None: - debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) - return False + for host_platform, target_platform in host_target_list: + + debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - cl_path = os.path.join(vc_dir, 'Tools','MSVC', vc_specific_version, 'bin', host_trgt_dir[0], host_trgt_dir[1], _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) - if os.path.exists(cl_path): - debug('found %s!', _CL_EXE_NAME) - return True - - elif host_platform == "amd64" and host_trgt_dir[0] == "Hostx64": - # Special case: fallback to Hostx86 if Hostx64 was tried - # and failed. This is because VS 2017 Express running on amd64 - # will look to our probe like the host dir should be Hostx64, - # but Express uses Hostx86 anyway. - # We should key this off the "x86_amd64" and related pseudo - # targets, but we don't see those in this function. - host_trgt_dir = ("Hostx86", host_trgt_dir[1]) - cl_path = os.path.join(vc_dir, 'Tools','MSVC', vc_specific_version, 'bin', host_trgt_dir[0], host_trgt_dir[1], _CL_EXE_NAME) + batchfile_clpath = _GE2017_HOST_TARGET_BATCHFILE_CLPATH.get((host_platform, target_platform), None) + if batchfile_clpath is None: + debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) + continue + + _, clpath = batchfile_clpath + cl_path = os.path.join(vc_dir, 'Tools', 'MSVC', vc_specific_version, clpath, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + if os.path.exists(cl_path): debug('found %s!', _CL_EXE_NAME) return True - elif 14 >= ver_num >= 8: - # Set default value to be -1 as "", which is the value for x86/x86, - # yields true when tested if not host_trgt_dir - host_trgt_dir = _HOST_TARGET_TO_CL_DIR.get((host_platform, target_platform), None) - if host_trgt_dir is None: - debug('unsupported host/target platform combo') - return False + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) - cl_path = os.path.join(vc_dir, 'bin', host_trgt_dir, _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + for host_platform, target_platform in host_target_list: - cl_path_exists = os.path.exists(cl_path) - if not cl_path_exists and host_platform == 'amd64': - # older versions of visual studio only had x86 binaries, - # so if the host platform is amd64, we need to check cross - # compile options (x86 binary compiles some other target on a 64 bit os) + debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - # Set default value to be -1 as "" which is the value for x86/x86 yields true when tested - # if not host_trgt_dir - host_trgt_dir = _HOST_TARGET_TO_CL_DIR.get(('x86', target_platform), None) - if host_trgt_dir is None: - return False + batcharg_clpath = _LE2015_HOST_TARGET_BATCHARG_CLPATH.get((host_platform, target_platform), None) + if batcharg_clpath is None: + debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) + continue - cl_path = os.path.join(vc_dir, 'bin', host_trgt_dir, _CL_EXE_NAME) + _, clpath = batcharg_clpath + cl_path = os.path.join(vc_dir, clpath, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) - cl_path_exists = os.path.exists(cl_path) - if cl_path_exists: - debug('found %s', _CL_EXE_NAME) - return True + if os.path.exists(cl_path): + debug('found %s', _CL_EXE_NAME) + return True + + elif 8 > vernum >= 6: + # 7.1 (VS2003) to 6.0 (VS6) - elif 8 > ver_num >= 6: # quick check for vc_dir/bin and vc_dir/ before walk # need to check root as the walk only considers subdirectories for cl_dir in ('bin', ''): @@ -675,9 +810,10 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('%s found %s', _CL_EXE_NAME, cl_path) return True return False + else: # version not support return false - debug('unsupported MSVC version: %s', str(ver_num)) + debug('unsupported MSVC version: %s', str(vernum)) return False @@ -735,21 +871,35 @@ def reset_installed_vcs(): # of time not parsing every time. script_env_cache = None +re_output_error = None def script_env(script, args=None): global script_env_cache + global re_output_error if script_env_cache is None: script_env_cache = common.read_script_env_cache() cache_key = "{}--{}".format(script, args) cache_data = script_env_cache.get(cache_key, None) if cache_data is None: + if re_output_error is None: + re_output_error = re.compile('^(' + '|'.join([ + 'VSINSTALLDIR variable is not set', # 2002-2003 + 'The specified configuration type is missing', # 2005+ + 'Error in script usage', # 2005+ + 'ERROR\:', # 2005+ + '\!ERROR\!', # 2015-2015 + '\[ERROR\:', # 2017+ + '\[ERROR\]', # 2017+ + 'Syntax\:', # 2017+ + ]) + ')') + stdout = common.get_output(script, args) # Stupid batch files do not set return code: we take a look at the # beginning of the output for an error message instead olines = stdout.splitlines() - if olines[0].startswith("The specified configuration type is missing"): + if re_output_error.match(olines[0]): raise BatchFileExecutionError("\n".join(olines[:2])) cache_data = common.parse_output(stdout) @@ -816,57 +966,20 @@ def msvc_find_valid_batch_script(env, version): get it right. """ - # Find the host, target, and if present the requested target: - platforms = get_host_target(env) - debug("host_platform %s, target_platform %s req_target_platform %s", *platforms) - host_platform, target_platform, req_target_platform = platforms - - # Most combinations of host + target are straightforward. - # While all MSVC / Visual Studio tools are pysically 32-bit, they - # make it look like there are 64-bit tools if the host is 64-bit, - # so you can invoke the environment batch script to set up to build, - # say, amd64 host -> x86 target. Express versions are an exception: - # they always look 32-bit, so the batch scripts with 64-bit - # host parts are absent. We try to fix that up in a couple of ways. - # One is here: we make a table of "targets" to try, with the extra - # targets being tags that tell us to try a different "host" instead - # of the deduced host. - try_target_archs = [target_platform] - if req_target_platform in ('amd64', 'x86_64'): - try_target_archs.append('x86_amd64') - elif req_target_platform in ('x86',): - try_target_archs.append('x86_x86') - elif req_target_platform in ('arm',): - try_target_archs.append('x86_arm') - elif req_target_platform in ('arm64',): - try_target_archs.append('x86_arm64') - elif not req_target_platform: - if target_platform in ('amd64', 'x86_64'): - try_target_archs.append('x86_amd64') - # If the user hasn't specifically requested a TARGET_ARCH, - # and the TARGET_ARCH is amd64 then also try 32 bits - # if there are no viable 64 bit tools installed - try_target_archs.append('x86') - - debug("host_platform: %s, try_target_archs: %s", host_platform, try_target_archs) + # Find the host, target, and all candidate (host, target) platform combinations: + platforms = get_host_target(env, version) + debug("host_platform %s, target_platform %s host_target_list %s", *platforms) + host_platform, target_platform, host_target_list = platforms d = None - for tp in try_target_archs: + for host_arch, target_arch, in host_target_list: # Set to current arch. - env['TARGET_ARCH'] = tp - - debug("trying target_platform:%s", tp) - host_target = (host_platform, tp) - if not is_host_target_supported(host_target, version): - warn_msg = "host, target = %s not supported for MSVC version %s" % \ - (host_target, version) - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) - arg = _HOST_TARGET_ARCH_TO_BAT_ARCH[host_target] + env['TARGET_ARCH'] = target_arch # Try to locate a batch file for this host/target platform combo try: - (vc_script, use_arg, sdk_script) = find_batch_file(env, version, host_platform, tp) - debug('vc_script:%s sdk_script:%s', vc_script, sdk_script) + (vc_script, arg, sdk_script) = find_batch_file(env, version, host_arch, target_arch) + debug('vc_script:%s vc_script_arg:%s sdk_script:%s', vc_script, arg, sdk_script) except VisualCException as e: msg = str(e) debug('Caught exception while looking for batch file (%s)', msg) @@ -881,8 +994,6 @@ def msvc_find_valid_batch_script(env, version): debug('use_script 2 %s, args:%s', repr(vc_script), arg) found = None if vc_script: - if not use_arg: - arg = '' # bat file will supply platform type # Get just version numbers maj, min = msvc_version_to_maj_min(version) # VS2015+ @@ -916,7 +1027,7 @@ def msvc_find_valid_batch_script(env, version): # If we cannot find a viable installed compiler, reset the TARGET_ARCH # To it's initial value if not d: - env['TARGET_ARCH']=req_target_platform + env['TARGET_ARCH'] = target_platform return d diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index eb09def..04ea808 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -114,13 +114,13 @@ class MSVcTestCase(unittest.TestCase): check = SCons.Tool.MSCommon.vc._check_cl_exists_in_vc_dir env={'TARGET_ARCH':'x86'} - p = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR[('x86','x86')] - MSVcTestCase._createDummyCl(p) + _, p = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATH[('x86','x86')] + MSVcTestCase._createDummyCl(p, add_bin=False) # print("retval:%s"%check(env, '.', '8.0')) - # Setup for VC 14+ tests + # Setup for 14.1 (VS2017) and later tests # Create the VC minor/major version file tools_version_file = SCons.Tool.MSCommon.vc._VC_TOOLS_VERSION_FILE @@ -134,25 +134,26 @@ class MSVcTestCase(unittest.TestCase): print("Failed trying to write :%s :%s"%(tools_version_file, e)) - # Now walk all the valid combinations of host/target for VC 14 + - vc_gt_14_map = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR_GREATER_THAN_14 + # Now walk all the valid combinations of host/target for 14.1 (VS2017) and later + vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_ALL_TARGETS_MAP['_all_'] - for key, value in vc_gt_14_map.items(): - # print("GT 14 Got: %s -> %s"%(key,value)) + for host, target in vc_ge2017_list: + batfile, clpath = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATH[(host,target)] + # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpath)) - env={'TARGET_ARCH':key[1], 'HOST_ARCH':key[0]} - path = os.path.join('.','Tools','MSVC', MS_TOOLS_VERSION, 'bin', value[0], value[1]) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.','Tools','MSVC', MS_TOOLS_VERSION, clpath) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '14.1') - # print("for:%s got :%s"%(key[1], result)) - self.assertTrue(result, "Checking host: %s target: %s"%(value[0], value[1])) + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(value[0], value[1])) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedHostArch: pass else: @@ -163,22 +164,24 @@ class MSVcTestCase(unittest.TestCase): try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(value[0], value[1])) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['TARGET_ARCH']) - - # Test >8 < 14 VC versions - vc_map = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR - for key,value in vc_map.items(): - # print("LT 14 Got: %s -> %s"%(key,value)) - env={'TARGET_ARCH':key[1], 'HOST_ARCH':key[0]} - path = os.path.join('.', 'bin', value ) + self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) + + # Test 14.0 (VS2015) to 8.0 (VS2005) versions + vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_ALL_TARGETS_MAP['_all_'] + + for host, target in vc_le2015_list: + batarg, clpath = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATH[(host, target)] + # print("LE 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpath)) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.', clpath) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '9.0') - # print("for:%s got :%s"%(key[1], result)) - self.assertTrue(result, "Checking host: %s target: %s"%(key[0], key[1])) + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} @@ -200,7 +203,41 @@ class MSVcTestCase(unittest.TestCase): except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['TARGET_ARCH']) + self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) + + # Test 7.1 (VS2003) and earlier + vc_le2003_list = SCons.Tool.MSCommon.vc._LE2003_HOST_ALL_TARGETS_MAP['_all_'] + + for host, target in vc_le2003_list: + # print("LE 7.1 Got: (%s, %s)"%(host,target)) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.') + MSVcTestCase._createDummyCl(path) + result=check(env, '.', '6.0') + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) + + # Now test bogus value for HOST_ARCH + env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} + try: + result=check(env, '.', '6.0') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedHostArch: + pass + else: + self.fail('Did not fail when HOST_ARCH specified as: %s'%env['HOST_ARCH']) + + # Now test bogus value for TARGET_ARCH + env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} + try: + result=check(env, '.', '6.0') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedTargetArch: + pass + else: + self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) -- cgit v0.12 From e7ad5bb7625ac85cf79967d19245b11dadd6ec52 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 7 Apr 2022 08:57:51 -0400 Subject: Refactor parallel dictionaries to namedtuple. Guarantee all hosts dictionary is ordered via OrderedDict. Bug fix if target/host map construction. Adjust all targets lists to have native host first in list. --- SCons/Tool/MSCommon/vc.py | 294 +++++++++++++++++++++++------------------ SCons/Tool/MSCommon/vcTests.py | 6 +- 2 files changed, 166 insertions(+), 134 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index d9160a9..54b4184 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -46,6 +46,10 @@ import platform from string import digits as string_digits from subprocess import PIPE import re +from collections import ( + namedtuple, + OrderedDict, +) import SCons.Util import SCons.Warnings @@ -97,57 +101,98 @@ _ARCH_TO_CANONICAL = { "aarch64" : "arm64", } -# utility methods for derived data structure construction - -def _make_host_target_map(all_hosts, all_targets): - # host_target_map[host][target] -> host_target_list - host_target_map = {} - for host, host_list in all_hosts.items(): - host_target_map[host] = {} - for host_platform in host_list: - for target_platform in all_targets[host_platform]: - if target_platform not in host_target_map[host]: - host_target_map[host][target_platform] = [] - host_target_map[host][target_platform].append((host_platform, target_platform)) - return host_target_map - -def _make_host_all_targets_map(host_target_map, all_targets): - # host_all_target_map[host] -> host_target_list - # special host key '_all_' contains all (host,target) combinations - all = '_all_' - host_all_target_map = {} - host_all_target_map[all] = [] - for host in host_target_map.keys(): - host_all_target_map[host] = [] - for target in all_targets[host]: - for host_target in host_target_map[host][target]: - for key in (host, all): - if host_target not in host_all_target_map[key]: - host_all_target_map[key].append(host_target) - return host_all_target_map - -def _make_host_default_target_map(host_target_map, default_targets): - # host_default_target_map[host] -> host_target_list - host_default_target_map = {} - for host in host_target_map.keys(): - host_default_target_map[host] = [] - for target in default_targets[host]: - host_default_target_map[host].extend(host_target_map[host][target]) - return host_default_target_map - -def _make_target_host_map(host_target_map): - # target_host_map[target][host] -> host_target_list - target_host_map = {} - for host in host_target_map.keys(): - for target in host_target_map[host].keys(): - if target not in target_host_map: - target_host_map[target] = {} - if host not in target_host_map[target]: - target_host_map[target][host] = [] - for host_target in host_target_map[host][target]: - if host_target not in target_host_map[target][host]: - target_host_map[target][host].append(host_target) - return target_host_map +# host/target candidate lists +_HOST_TARGET_CONFIG_NT = namedtuple("HostTargetConfig", [ + # defined + "label", # name for debugging/output + "host_all_hosts", # host_all_hosts[host] -> host_list + "host_all_targets", # host_all_targets[host] -> target_list + "host_def_targets", # host_def_targets[host] -> target_list + # derived + "all_pairs", # host_target_list + "host_target_map", # host_target_map[host][target] -> host_target_list + "host_all_targets_map", # host_all_targets_map[host][target] -> host_target_list + "host_def_targets_map", # host_def_targets_map[host][target] -> host_target_list + "target_host_map", # target_host_map[target][host] -> host_target_list + ]) + +def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host_def_targets): + + def _make_host_target_map(all_hosts, all_targets): + # host_target_map[host][target] -> host_target_list + host_target_map = {} + for host, host_list in all_hosts.items(): + host_target_map[host] = {} + for host_platform in host_list: + for target_platform in all_targets[host_platform]: + if target_platform not in host_target_map[host]: + host_target_map[host][target_platform] = [] + host_target_map[host][target_platform].append((host_platform, target_platform)) + return host_target_map + + def _make_host_all_targets_map(all_hosts, host_target_map, all_targets): + # host_all_target_map[host] -> host_target_list + # special host key '_all_' contains all (host,target) combinations + all = '_all_' + host_all_targets_map = {} + host_all_targets_map[all] = [] + for host, host_list in all_hosts.items(): + host_all_targets_map[host] = [] + for host_platform in host_list: + for target in all_targets[host_platform]: + for host_target in host_target_map[host_platform][target]: + for host_key in (host, all): + if host_target not in host_all_targets_map[host_key]: + host_all_targets_map[host_key].append(host_target) + return host_all_targets_map + + def _make_host_def_targets_map(all_hosts, host_target_map, def_targets): + # host_def_targets_map[host] -> host_target_list + host_def_targets_map = {} + for host, host_list in all_hosts.items(): + host_def_targets_map[host] = [] + for host_platform in host_list: + for target in def_targets[host_platform]: + for host_target in host_target_map[host_platform][target]: + if host_target not in host_def_targets_map[host]: + host_def_targets_map[host].append(host_target) + return host_def_targets_map + + def _make_target_host_map(all_hosts, host_all_targets_map): + # target_host_map[target][host] -> host_target_list + target_host_map = {} + for host_platform in all_hosts.keys(): + for host_target in host_all_targets_map[host_platform]: + _, target = host_target + if target not in target_host_map: + target_host_map[target] = {} + if host_platform not in target_host_map[target]: + target_host_map[target][host_platform] = [] + if host_target not in target_host_map[target][host_platform]: + target_host_map[target][host_platform].append(host_target) + return target_host_map + + host_target_map = _make_host_target_map(host_all_hosts, host_all_targets) + host_all_targets_map = _make_host_all_targets_map(host_all_hosts, host_target_map, host_all_targets) + host_def_targets_map = _make_host_def_targets_map(host_all_hosts, host_target_map, host_def_targets) + target_host_map = _make_target_host_map(host_all_hosts, host_all_targets_map) + + all_pairs = host_all_targets_map['_all_'] + del host_all_targets_map['_all_'] + + host_target_cfg = _HOST_TARGET_CONFIG_NT( + label = label, + host_all_hosts = dict(host_all_hosts), + host_all_targets = host_all_targets, + host_def_targets = host_def_targets, + all_pairs = all_pairs, + host_target_map = host_target_map, + host_all_targets_map = host_all_targets_map, + host_def_targets_map = host_def_targets_map, + target_host_map = target_host_map, + ) + + return host_target_cfg # 14.1 (VS2017) and later @@ -164,9 +209,6 @@ def _make_target_host_map(host_target_map): # version directory. For example, /VC/Tools/MSVC/14.31.31103/bin/Hostx64/x64. # The cl path fragment under the toolset version folder is the second value of # the stored tuple. -# -# Note: 2017 Express uses Hostx86 even if it's on 64-bit Windows which is not -# reflected in this table. _GE2017_HOST_TARGET_BATCHFILE_CLPATH = { @@ -182,30 +224,28 @@ _GE2017_HOST_TARGET_BATCHFILE_CLPATH = { } -_GE2017_HOST_ALL_HOSTS = { - 'amd64': ['amd64', 'x86'], - 'x86': ['x86'], -} +_GE2017_HOST_TARGET_CFG = _host_target_config_factory( -_GE2017_HOST_ALL_TARGETS = { - 'amd64': ['amd64', 'x86', 'arm', 'arm64'], - 'x86': ['amd64', 'x86', 'arm', 'arm64'], -} + label = 'GE2017', -_GE2017_HOST_DEFAULT_TARGETS = { - 'amd64': ['amd64', 'x86'], - 'x86': ['x86'], -} + host_all_hosts = OrderedDict([ + ('amd64', ['amd64', 'x86']), + ('x86', ['x86']), + ]), + + host_all_targets = { + 'amd64': ['amd64', 'x86', 'arm', 'arm64'], + 'x86': ['x86', 'amd64', 'arm', 'arm64'], + }, -_GE2017_HOST_TARGET_MAP = _make_host_target_map(_GE2017_HOST_ALL_HOSTS, _GE2017_HOST_ALL_TARGETS) -_GE2017_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_GE2017_HOST_TARGET_MAP, _GE2017_HOST_ALL_TARGETS) -_GE2017_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_GE2017_HOST_TARGET_MAP, _GE2017_HOST_DEFAULT_TARGETS) -_GE2017_TARGET_HOST_MAP = _make_target_host_map(_GE2017_HOST_TARGET_MAP) + host_def_targets = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + }, -#debug("_GE2017_HOST_TARGET_MAP: %s", _GE2017_HOST_TARGET_MAP) -#debug("_GE2017_HOST_ALL_TARGETS_MAP: %s", _GE2017_HOST_ALL_TARGETS_MAP) -#debug("_GE2017_HOST_DEF_TARGETS_MAP: %s", _GE2017_HOST_DEF_TARGETS_MAP) -#debug("_GE2017_TARGET_HOST_MAP: %s", _GE2017_TARGET_HOST_MAP) +) + +# debug("_GE2017_HOST_TARGET_CFG: %s", _GE2017_HOST_TARGET_CFG) # 14.0 (VS2015) to 8.0 (VS2005) @@ -232,66 +272,62 @@ _LE2015_HOST_TARGET_BATCHARG_CLPATH = { } -_LE2015_HOST_ALL_HOSTS = { - 'amd64': ['amd64', 'x86'], - 'x86': ['x86'], - 'arm': ['arm'], - 'ia64': ['ia64'], -} +_LE2015_HOST_TARGET_CFG = _host_target_config_factory( -_LE2015_HOST_ALL_TARGETS = { - 'amd64': ['amd64', 'x86', 'arm'], - 'x86': ['amd64', 'x86', 'arm', 'ia64'], - 'arm': ['arm'], - 'ia64': ['ia64'], -} + label = 'LE2015', -_LE2015_HOST_DEFAULT_TARGETS = { - 'amd64': ['amd64', 'x86'], - 'x86': ['x86'], - 'arm': ['arm'], - 'ia64': ['ia64'], -} + host_all_hosts = OrderedDict([ + ('amd64', ['amd64', 'x86']), + ('x86', ['x86']), + ('arm', ['arm']), + ('ia64', ['ia64']), + ]), -_LE2015_HOST_TARGET_MAP = _make_host_target_map(_LE2015_HOST_ALL_HOSTS, _LE2015_HOST_ALL_TARGETS) -_LE2015_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_LE2015_HOST_TARGET_MAP, _LE2015_HOST_ALL_TARGETS) -_LE2015_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_LE2015_HOST_TARGET_MAP, _LE2015_HOST_DEFAULT_TARGETS) -_LE2015_TARGET_HOST_MAP = _make_target_host_map(_LE2015_HOST_TARGET_MAP) + host_all_targets = { + 'amd64': ['amd64', 'x86', 'arm'], + 'x86': ['x86', 'amd64', 'arm', 'ia64'], + 'arm': ['arm'], + 'ia64': ['ia64'], + }, -#debug("_LE2015_HOST_TARGET_MAP: %s", _LE2015_HOST_TARGET_MAP) -#debug("_LE2015_HOST_ALL_TARGETS_MAP: %s", _LE2015_HOST_ALL_TARGETS_MAP) -#debug("_LE2015_HOST_DEF_TARGETS_MAP: %s", _LE2015_HOST_DEF_TARGETS_MAP) -#debug("_LE2015_TARGET_HOST_MAP: %s", _LE2015_TARGET_HOST_MAP) + host_def_targets = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm': ['arm'], + 'ia64': ['ia64'], + }, + +) + +# debug("_LE2015_HOST_TARGET_CFG: %s", _LE2015_HOST_TARGET_CFG) # 7.1 (VS2003) and earlier # For 7.1 (VS2003) and earlier, there are only x86 targets and the batch files # take no arguments. -_LE2003_HOST_ALL_HOSTS = { - 'amd64': ['x86'], - 'x86': ['x86'], -} +_LE2003_HOST_TARGET_CFG = _host_target_config_factory( -_LE2003_HOST_ALL_TARGETS = { - 'amd64': ['x86'], - 'x86': ['x86'], -} + label = 'LE2003', -_LE2003_HOST_DEFAULT_TARGETS = { - 'amd64': ['x86'], - 'x86': ['x86'], -} + host_all_hosts = OrderedDict([ + ('amd64', ['x86']), + ('x86', ['x86']), + ]), + + host_all_targets = { + 'amd64': ['x86'], + 'x86': ['x86'], + }, -_LE2003_HOST_TARGET_MAP = _make_host_target_map(_LE2003_HOST_ALL_HOSTS, _LE2003_HOST_ALL_TARGETS) -_LE2003_HOST_ALL_TARGETS_MAP = _make_host_all_targets_map(_LE2003_HOST_TARGET_MAP, _LE2003_HOST_ALL_TARGETS) -_LE2003_HOST_DEF_TARGETS_MAP = _make_host_default_target_map(_LE2003_HOST_TARGET_MAP, _LE2003_HOST_DEFAULT_TARGETS) -_LE2003_TARGET_HOST_MAP = _make_target_host_map(_LE2003_HOST_TARGET_MAP) + host_def_targets = { + 'amd64': ['x86'], + 'x86': ['x86'], + }, -#debug("_LE2003_HOST_TARGET_MAP: %s", _LE2003_HOST_TARGET_MAP) -#debug("_LE2003_HOST_ALL_TARGETS_MAP: %s", _LE2003_HOST_ALL_TARGETS_MAP) -#debug("_LE2003_HOST_DEF_TARGETS_MAP: %s", _LE2003_HOST_DEF_TARGETS_MAP) -#debug("_LE2003_TARGET_HOST_MAP: %s", _LE2003_TARGET_HOST_MAP) +) + +# debug("_LE2003_HOST_TARGET_CFG: %s", _LE2003_HOST_TARGET_CFG) _CL_EXE_NAME = 'cl.exe' @@ -347,19 +383,13 @@ def get_host_target(env, msvc_version, all_host_targets=False): if vernum > 14: # 14.1 (VS2017) and later - target_host_map = _GE2017_TARGET_HOST_MAP - host_all_targets_map = _GE2017_HOST_ALL_TARGETS_MAP - host_def_targets_map = _GE2017_HOST_DEF_TARGETS_MAP + host_target_cfg = _GE2017_HOST_TARGET_CFG elif 14 >= vernum >= 8: # 14.0 (VS2015) to 8.0 (VS2005) - target_host_map = _LE2015_TARGET_HOST_MAP - host_all_targets_map = _LE2015_HOST_ALL_TARGETS_MAP - host_def_targets_map = _LE2003_HOST_DEF_TARGETS_MAP + host_target_cfg = _LE2015_HOST_TARGET_CFG else: # 7.1 (VS2003) and earlier - target_host_map = _LE2003_TARGET_HOST_MAP - host_all_targets_map = _LE2015_HOST_ALL_TARGETS_MAP - host_def_targets_map = _LE2003_HOST_DEF_TARGETS_MAP + host_target_cfg = _LE2003_HOST_TARGET_CFG host_arch = env.get('HOST_ARCH') if env else None debug("HOST_ARCH:%s", str(host_arch)) @@ -383,6 +413,8 @@ def get_host_target(env, msvc_version, all_host_targets=False): % (repr(target_arch), all_archs) ) from None + target_host_map = host_target_cfg.target_host_map + try: host_target_list = target_host_map[target_platform][host_platform] except KeyError: @@ -398,9 +430,9 @@ def get_host_target(env, msvc_version, all_host_targets=False): target_platform = None if all_host_targets: - host_targets_map = host_all_targets_map + host_targets_map = host_target_cfg.host_all_targets_map else: - host_targets_map = host_def_targets_map + host_targets_map = host_target_cfg.host_def_targets_map try: host_target_list = host_targets_map[host_platform] diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index 04ea808..a5dceae 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -135,7 +135,7 @@ class MSVcTestCase(unittest.TestCase): # Now walk all the valid combinations of host/target for 14.1 (VS2017) and later - vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_ALL_TARGETS_MAP['_all_'] + vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_CFG.all_pairs for host, target in vc_ge2017_list: batfile, clpath = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATH[(host,target)] @@ -171,7 +171,7 @@ class MSVcTestCase(unittest.TestCase): self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) # Test 14.0 (VS2015) to 8.0 (VS2005) versions - vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_ALL_TARGETS_MAP['_all_'] + vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_CFG.all_pairs for host, target in vc_le2015_list: batarg, clpath = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATH[(host, target)] @@ -206,7 +206,7 @@ class MSVcTestCase(unittest.TestCase): self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) # Test 7.1 (VS2003) and earlier - vc_le2003_list = SCons.Tool.MSCommon.vc._LE2003_HOST_ALL_TARGETS_MAP['_all_'] + vc_le2003_list = SCons.Tool.MSCommon.vc._LE2003_HOST_TARGET_CFG.all_pairs for host, target in vc_le2003_list: # print("LE 7.1 Got: (%s, %s)"%(host,target)) -- cgit v0.12 From a24e29f4413772408a752248cd707ad66d91d8ca Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:07:04 -0400 Subject: Fix to recognize Visual C++ For Python (9.0): registry key returns installation directory and not vc product directory so cl.exe is not detected and 9.0 does not appear in installed vcs list. --- SCons/Tool/MSCommon/vc.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 54b4184..7190033 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -807,6 +807,11 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): elif 14 >= vernum >= 8: # 14.0 (VS2015) to 8.0 (VS2005) + clpath_prefixes = [''] + if msvc_version == '9.0': + # Visual C++ for Python registry key is installdir (root) not productdir (vc) + clpath_prefixes.append('VC') + for host_platform, target_platform in host_target_list: debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) @@ -817,12 +822,15 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): continue _, clpath = batcharg_clpath - cl_path = os.path.join(vc_dir, clpath, _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + for clpath_prefix in clpath_prefixes: - if os.path.exists(cl_path): - debug('found %s', _CL_EXE_NAME) - return True + clpath_adj = os.path.join(clpath_prefix, clpath) if clpath_prefix else clpath + cl_path = os.path.join(vc_dir, clpath_adj, _CL_EXE_NAME) + debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + + if os.path.exists(cl_path): + debug('found %s', _CL_EXE_NAME) + return True elif 8 > vernum >= 6: # 7.1 (VS2003) to 6.0 (VS6) -- cgit v0.12 From b340c918ccfb869fe2fbb894bd8b11bbcb0be71f Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 8 Apr 2022 12:59:51 -0400 Subject: Change the string path fragments to tuples of path components. Change the constant and working variables to be consistent with change in data type (readability). Update the tests accordingly. --- SCons/Tool/MSCommon/vc.py | 70 +++++++++++++++++++++--------------------- SCons/Tool/MSCommon/vcTests.py | 17 +++++----- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 7190033..d01f5eb 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -197,7 +197,7 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host # 14.1 (VS2017) and later # Given a (host, target) tuple, return a tuple containing the batch file to -# look for and the path fragment to find cl.exe. We can't rely on returning +# look for and a tuple of path components to find cl.exe. We can't rely on returning # an arg to use for vcvarsall.bat, because that script will run even if given # a host/target pair that isn't installed. # @@ -210,17 +210,17 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host # The cl path fragment under the toolset version folder is the second value of # the stored tuple. -_GE2017_HOST_TARGET_BATCHFILE_CLPATH = { +_GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { - ('amd64', 'amd64') : ( 'vcvars64.bat', r'bin\Hostx64\x64' ), - ('amd64', 'x86') : ( 'vcvarsamd64_x86.bat', r'bin\Hostx64\x86' ), - ('amd64', 'arm') : ( 'vcvarsamd64_arm.bat', r'bin\Hostx64\arm' ), - ('amd64', 'arm64') : ( 'vcvarsamd64_arm64.bat', r'bin\Hostx64\arm64' ), + ('amd64', 'amd64') : ( 'vcvars64.bat', ('bin', 'Hostx64', 'x64') ), + ('amd64', 'x86') : ( 'vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86') ), + ('amd64', 'arm') : ( 'vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm') ), + ('amd64', 'arm64') : ( 'vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64') ), - ('x86', 'amd64') : ( 'vcvarsx86_amd64.bat', r'bin\Hostx86\x64' ), - ('x86', 'x86') : ( 'vcvars32.bat', r'bin\Hostx86\x86' ), - ('x86', 'arm') : ( 'vcvarsx86_arm.bat', r'bin\Hostx86\arm' ), - ('x86', 'arm64') : ( 'vcvarsx86_arm64.bat', r'bin\Hostx86\arm64' ), + ('x86', 'amd64') : ( 'vcvarsx86_amd64.bat', ('bin', 'Hostx86', 'x64') ), + ('x86', 'x86') : ( 'vcvars32.bat', ('bin', 'Hostx86', 'x86') ), + ('x86', 'arm') : ( 'vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm') ), + ('x86', 'arm64') : ( 'vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64') ), } @@ -250,25 +250,25 @@ _GE2017_HOST_TARGET_CFG = _host_target_config_factory( # 14.0 (VS2015) to 8.0 (VS2005) # Given a (host, target) tuple, return a tuple containing the argument for -# the batch file and the the path fragment to find cl.exe. +# the batch file and a tuple of the path components to find cl.exe. # # In 14.0 (VS2015) and earlier, the original x86 tools are in the tools # bin directory (i.e., /VC/bin). Any other tools are in subdirectory # named for the the host/target pair or a single name if the host==target. -_LE2015_HOST_TARGET_BATCHARG_CLPATH = { +_LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS = { - ('amd64', 'amd64') : ( 'amd64', r'bin\amd64' ), - ('amd64', 'x86') : ( 'amd64_x86', r'bin\amd64_x86' ), - ('amd64', 'arm') : ( 'amd64_arm', r'bin\amd64_arm' ), + ('amd64', 'amd64') : ( 'amd64', ('bin', 'amd64') ), + ('amd64', 'x86') : ( 'amd64_x86', ('bin', 'amd64_x86') ), + ('amd64', 'arm') : ( 'amd64_arm', ('bin', 'amd64_arm') ), - ('x86', 'amd64') : ( 'x86_amd64', r'bin\x86_amd64' ), - ('x86', 'x86') : ( 'x86', r'bin' ), - ('x86', 'arm') : ( 'x86_arm', r'bin\x86_arm' ), - ('x86', 'ia64') : ( 'x86_ia64', r'bin\x86_ia64' ), + ('x86', 'amd64') : ( 'x86_amd64', ('bin', 'x86_amd64') ), + ('x86', 'x86') : ( 'x86', ('bin', ) ), + ('x86', 'arm') : ( 'x86_arm', ('bin', 'x86_arm') ), + ('x86', 'ia64') : ( 'x86_ia64', ('bin', 'x86_ia64') ), - ('arm', 'arm') : ( 'arm', r'bin\arm' ), - ('ia64', 'ia64') : ( 'ia64', r'bin\ia64' ), + ('arm', 'arm') : ( 'arm', ('bin', 'arm') ), + ('ia64', 'ia64') : ( 'ia64', ('bin', 'ia64') ), } @@ -703,11 +703,11 @@ def find_batch_file(env, msvc_version, host_arch, target_arch): if vernum > 14: # 14.1 (VS2017) and later batfiledir = os.path.join(pdir, "Auxiliary", "Build") - batfile, _ = _GE2017_HOST_TARGET_BATCHFILE_CLPATH[(host_arch, target_arch)] + batfile, _ = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host_arch, target_arch)] batfilename = os.path.join(batfiledir, batfile) elif 14 >= vernum >= 8: # 14.0 (VS2015) to 8.0 (VS2005) - arg, _ = _LE2015_HOST_TARGET_BATCHARG_CLPATH[(host_arch, target_arch)] + arg, _ = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host_arch, target_arch)] batfilename = os.path.join(pdir, "vcvarsall.bat") elif 8 > vernum >= 7: # 7.1 (VS2003) to 7.0 (VS2003) @@ -791,13 +791,13 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - batchfile_clpath = _GE2017_HOST_TARGET_BATCHFILE_CLPATH.get((host_platform, target_platform), None) - if batchfile_clpath is None: + batchfile_clpathcomps = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS.get((host_platform, target_platform), None) + if batchfile_clpathcomps is None: debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) continue - _, clpath = batchfile_clpath - cl_path = os.path.join(vc_dir, 'Tools', 'MSVC', vc_specific_version, clpath, _CL_EXE_NAME) + _, cl_path_comps = batchfile_clpathcomps + cl_path = os.path.join(vc_dir, 'Tools', 'MSVC', vc_specific_version, *cl_path_comps, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) if os.path.exists(cl_path): @@ -807,25 +807,25 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): elif 14 >= vernum >= 8: # 14.0 (VS2015) to 8.0 (VS2005) - clpath_prefixes = [''] + cl_path_prefixes = [None] if msvc_version == '9.0': # Visual C++ for Python registry key is installdir (root) not productdir (vc) - clpath_prefixes.append('VC') + cl_path_prefixes.append( ('VC',) ) for host_platform, target_platform in host_target_list: debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - batcharg_clpath = _LE2015_HOST_TARGET_BATCHARG_CLPATH.get((host_platform, target_platform), None) - if batcharg_clpath is None: + batcharg_clpathcomps = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS.get((host_platform, target_platform), None) + if batcharg_clpathcomps is None: debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) continue - _, clpath = batcharg_clpath - for clpath_prefix in clpath_prefixes: + _, cl_path_comps = batcharg_clpathcomps + for cl_path_prefix in cl_path_prefixes: - clpath_adj = os.path.join(clpath_prefix, clpath) if clpath_prefix else clpath - cl_path = os.path.join(vc_dir, clpath_adj, _CL_EXE_NAME) + cl_path_comps_adj = cl_path_prefix + cl_path_comps if cl_path_prefix else cl_path_comps + cl_path = os.path.join(vc_dir, *cl_path_comps_adj, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) if os.path.exists(cl_path): diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index a5dceae..6c146e4 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -114,8 +114,9 @@ class MSVcTestCase(unittest.TestCase): check = SCons.Tool.MSCommon.vc._check_cl_exists_in_vc_dir env={'TARGET_ARCH':'x86'} - _, p = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATH[('x86','x86')] - MSVcTestCase._createDummyCl(p, add_bin=False) + _, clpathcomps = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[('x86','x86')] + path = os.path.join('.', *clpathcomps) + MSVcTestCase._createDummyCl(path, add_bin=False) # print("retval:%s"%check(env, '.', '8.0')) @@ -138,11 +139,11 @@ class MSVcTestCase(unittest.TestCase): vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_CFG.all_pairs for host, target in vc_ge2017_list: - batfile, clpath = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATH[(host,target)] - # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpath)) + batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] + # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) env={'TARGET_ARCH':target, 'HOST_ARCH':host} - path = os.path.join('.','Tools','MSVC', MS_TOOLS_VERSION, clpath) + path = os.path.join('.', 'Tools', 'MSVC', MS_TOOLS_VERSION, *clpathcomps) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '14.1') # print("for:(%s, %s) got :%s"%(host, target, result)) @@ -174,10 +175,10 @@ class MSVcTestCase(unittest.TestCase): vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_CFG.all_pairs for host, target in vc_le2015_list: - batarg, clpath = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATH[(host, target)] - # print("LE 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpath)) + batarg, clpathcomps = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host, target)] + # print("LE 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpathcomps)) env={'TARGET_ARCH':target, 'HOST_ARCH':host} - path = os.path.join('.', clpath) + path = os.path.join('.', *clpathcomps) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '9.0') # print("for:(%s, %s) got :%s"%(host, target, result)) -- cgit v0.12 From be9db8b2a6f28ab40422d9f7116bfd9f0c0a10f3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 10 Apr 2022 08:05:58 -0400 Subject: Whitespace changes to appease sider. --- SCons/Tool/MSCommon/vc.py | 42 +++++++++++++++++++++--------------------- SCons/Tool/MSCommon/vcTests.py | 26 +++++++++++++------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index d01f5eb..0db710a 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -114,7 +114,7 @@ _HOST_TARGET_CONFIG_NT = namedtuple("HostTargetConfig", [ "host_all_targets_map", # host_all_targets_map[host][target] -> host_target_list "host_def_targets_map", # host_def_targets_map[host][target] -> host_target_list "target_host_map", # target_host_map[target][host] -> host_target_list - ]) +]) def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host_def_targets): @@ -212,15 +212,15 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { - ('amd64', 'amd64') : ( 'vcvars64.bat', ('bin', 'Hostx64', 'x64') ), - ('amd64', 'x86') : ( 'vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86') ), - ('amd64', 'arm') : ( 'vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm') ), - ('amd64', 'arm64') : ( 'vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64') ), + ('amd64', 'amd64') : ('vcvars64.bat', ('bin', 'Hostx64', 'x64')), + ('amd64', 'x86') : ('vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86')), + ('amd64', 'arm') : ('vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm')), + ('amd64', 'arm64') : ('vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64')), - ('x86', 'amd64') : ( 'vcvarsx86_amd64.bat', ('bin', 'Hostx86', 'x64') ), - ('x86', 'x86') : ( 'vcvars32.bat', ('bin', 'Hostx86', 'x86') ), - ('x86', 'arm') : ( 'vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm') ), - ('x86', 'arm64') : ( 'vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64') ), + ('x86', 'amd64') : ('vcvarsx86_amd64.bat', ('bin', 'Hostx86', 'x64')), + ('x86', 'x86') : ('vcvars32.bat', ('bin', 'Hostx86', 'x86')), + ('x86', 'arm') : ('vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm')), + ('x86', 'arm64') : ('vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64')), } @@ -258,17 +258,17 @@ _GE2017_HOST_TARGET_CFG = _host_target_config_factory( _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS = { - ('amd64', 'amd64') : ( 'amd64', ('bin', 'amd64') ), - ('amd64', 'x86') : ( 'amd64_x86', ('bin', 'amd64_x86') ), - ('amd64', 'arm') : ( 'amd64_arm', ('bin', 'amd64_arm') ), + ('amd64', 'amd64') : ('amd64', ('bin', 'amd64')), + ('amd64', 'x86') : ('amd64_x86', ('bin', 'amd64_x86')), + ('amd64', 'arm') : ('amd64_arm', ('bin', 'amd64_arm')), - ('x86', 'amd64') : ( 'x86_amd64', ('bin', 'x86_amd64') ), - ('x86', 'x86') : ( 'x86', ('bin', ) ), - ('x86', 'arm') : ( 'x86_arm', ('bin', 'x86_arm') ), - ('x86', 'ia64') : ( 'x86_ia64', ('bin', 'x86_ia64') ), + ('x86', 'amd64') : ('x86_amd64', ('bin', 'x86_amd64')), + ('x86', 'x86') : ('x86', ('bin', )), + ('x86', 'arm') : ('x86_arm', ('bin', 'x86_arm')), + ('x86', 'ia64') : ('x86_ia64', ('bin', 'x86_ia64')), - ('arm', 'arm') : ( 'arm', ('bin', 'arm') ), - ('ia64', 'ia64') : ( 'ia64', ('bin', 'ia64') ), + ('arm', 'arm') : ('arm', ('bin', 'arm')), + ('ia64', 'ia64') : ('ia64', ('bin', 'ia64')), } @@ -810,7 +810,7 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): cl_path_prefixes = [None] if msvc_version == '9.0': # Visual C++ for Python registry key is installdir (root) not productdir (vc) - cl_path_prefixes.append( ('VC',) ) + cl_path_prefixes.append(('VC',)) for host_platform, target_platform in host_target_list: @@ -824,7 +824,7 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): _, cl_path_comps = batcharg_clpathcomps for cl_path_prefix in cl_path_prefixes: - cl_path_comps_adj = cl_path_prefix + cl_path_comps if cl_path_prefix else cl_path_comps + cl_path_comps_adj = cl_path_prefix + cl_path_comps if cl_path_prefix else cl_path_comps cl_path = os.path.join(vc_dir, *cl_path_comps_adj, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) @@ -932,7 +932,7 @@ def script_env(script, args=None): '\[ERROR\:', # 2017+ '\[ERROR\]', # 2017+ 'Syntax\:', # 2017+ - ]) + ')') + ]) + ')') stdout = common.get_output(script, args) diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index 6c146e4..2b6fbe5 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -139,7 +139,7 @@ class MSVcTestCase(unittest.TestCase): vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_CFG.all_pairs for host, target in vc_ge2017_list: - batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] + batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) env={'TARGET_ARCH':target, 'HOST_ARCH':host} @@ -147,14 +147,14 @@ class MSVcTestCase(unittest.TestCase): MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '14.1') # print("for:(%s, %s) got :%s"%(host, target, result)) - self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedHostArch: pass else: @@ -165,11 +165,11 @@ class MSVcTestCase(unittest.TestCase): try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) # Test 14.0 (VS2015) to 8.0 (VS2005) versions vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_CFG.all_pairs @@ -182,7 +182,7 @@ class MSVcTestCase(unittest.TestCase): MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '9.0') # print("for:(%s, %s) got :%s"%(host, target, result)) - self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} @@ -193,7 +193,7 @@ class MSVcTestCase(unittest.TestCase): except MSVCUnsupportedHostArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['HOST_ARCH']) + self.fail('Did not fail when HOST_ARCH specified as: %s' % env['HOST_ARCH']) # Now test bogus value for TARGET_ARCH env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} @@ -204,7 +204,7 @@ class MSVcTestCase(unittest.TestCase): except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) # Test 7.1 (VS2003) and earlier vc_le2003_list = SCons.Tool.MSCommon.vc._LE2003_HOST_TARGET_CFG.all_pairs @@ -216,29 +216,29 @@ class MSVcTestCase(unittest.TestCase): MSVcTestCase._createDummyCl(path) result=check(env, '.', '6.0') # print("for:(%s, %s) got :%s"%(host, target, result)) - self.assertTrue(result, "Checking host: %s target: %s"%(host, target)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} try: result=check(env, '.', '6.0') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedHostArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['HOST_ARCH']) + self.fail('Did not fail when HOST_ARCH specified as: %s' % env['HOST_ARCH']) # Now test bogus value for TARGET_ARCH env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} try: result=check(env, '.', '6.0') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(env['HOST_ARCH'], env['TARGET_ARCH'])) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when TARGET_ARCH specified as: %s'%env['TARGET_ARCH']) + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) -- cgit v0.12 From 9c5813e5ed9a0c7e18b003094169be2974987171 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 10 Apr 2022 10:34:34 -0400 Subject: Change list of strings to list of raw strings when building regular expression. --- SCons/Tool/MSCommon/vc.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 0db710a..fa4c413 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -924,14 +924,14 @@ def script_env(script, args=None): if cache_data is None: if re_output_error is None: re_output_error = re.compile('^(' + '|'.join([ - 'VSINSTALLDIR variable is not set', # 2002-2003 - 'The specified configuration type is missing', # 2005+ - 'Error in script usage', # 2005+ - 'ERROR\:', # 2005+ - '\!ERROR\!', # 2015-2015 - '\[ERROR\:', # 2017+ - '\[ERROR\]', # 2017+ - 'Syntax\:', # 2017+ + r'VSINSTALLDIR variable is not set', # 2002-2003 + r'The specified configuration type is missing', # 2005+ + r'Error in script usage', # 2005+ + r'ERROR\:', # 2005+ + r'\!ERROR\!', # 2015-2015 + r'\[ERROR\:', # 2017+ + r'\[ERROR\]', # 2017+ + r'Syntax\:', # 2017+ ]) + ')') stdout = common.get_output(script, args) -- cgit v0.12 From d927ed8ff1ec74736f4b7f165a3f92a8745dff48 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 13 Apr 2022 15:46:03 -0400 Subject: Use the same mingw tool search paths in both the generate and exists methods. --- SCons/Tool/mingw.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/mingw.py b/SCons/Tool/mingw.py index 0d2bd6d..8e7ac2d 100644 --- a/SCons/Tool/mingw.py +++ b/SCons/Tool/mingw.py @@ -41,7 +41,7 @@ import SCons.Defaults import SCons.Tool import SCons.Util -mingw_paths = [ +mingw_base_paths = [ r'c:\MinGW\bin', r'C:\cygwin64\bin', r'C:\msys64', @@ -127,17 +127,24 @@ def find_version_specific_mingw_paths(): One example of default mingw install paths is: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev2\mingw64\bin - Use glob'ing to find such and add to mingw_paths + Use glob'ing to find such and add to mingw_base_paths """ new_paths = glob.glob(r"C:\mingw-w64\*\mingw64\bin") return new_paths +_mingw_all_paths = None + +def get_mingw_paths(): + global _mingw_all_paths + if _mingw_all_paths is None: + _mingw_all_paths = mingw_base_paths + find_version_specific_mingw_paths() + return _mingw_all_paths + def generate(env): - global mingw_paths # Check for reasoanble mingw default paths - mingw_paths += find_version_specific_mingw_paths() + mingw_paths = get_mingw_paths() mingw = SCons.Tool.find_program_path(env, key_program, default_paths=mingw_paths) if mingw: @@ -204,6 +211,7 @@ def generate(env): def exists(env): + mingw_paths = get_mingw_paths() mingw = SCons.Tool.find_program_path(env, key_program, default_paths=mingw_paths) if mingw: mingw_bin_dir = os.path.dirname(mingw) -- cgit v0.12 From ab05be34b4206d401fbdf312335d1e63889b7d61 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 14 Apr 2022 05:39:38 -0400 Subject: Update CHANGES.txt and RELEASE.txt. [skip ci] --- CHANGES.txt | 2 ++ RELEASE.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 8390f00..4521fb9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Joseph Brill: - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception when the user specified msvc script does not exist. + - Fix the mingw tool so that the generate and exists methods use the same mingw search paths + (issue #4134). From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message diff --git a/RELEASE.txt b/RELEASE.txt index be2444e..aee7491 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -58,6 +58,8 @@ FIXES with no defaults, so old usage where _defines() was called without source and target arguments would yield an exception. This issue was found via qt4 and qt5 tools in scons-contrib https://github.com/SCons/scons-contrib/issues/45 +- Fixed an inconsistency in the mingw tool search paths which could result in a mingw + installation not being detected (issue #4134). IMPROVEMENTS ------------ -- cgit v0.12 From 4bb4178b19294ddc0bc1ec801d28c0bb2509a4c5 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 Apr 2022 15:49:51 -0500 Subject: Add ninja alias option to label generated source files --- SCons/Tool/ninja/NinjaState.py | 74 +++++++++++++------ SCons/Tool/ninja/__init__.py | 6 +- SCons/Tool/ninja/ninja.xml | 14 +++- test/ninja/generated_sources_alias.py | 82 ++++++++++++++++++++++ test/ninja/ninja-fixture/gen_source.c | 9 +++ .../sconstruct_generated_sources_alias | 41 +++++++++++ 6 files changed, 199 insertions(+), 27 deletions(-) create mode 100644 test/ninja/generated_sources_alias.py create mode 100644 test/ninja/ninja-fixture/gen_source.c create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index fff60be..c9d8a89 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -360,26 +360,56 @@ class NinjaState: kwargs['pool'] = 'local_pool' ninja.rule(rule, **kwargs) - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) - - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files + # If the user supplied an alias to determine generated sources, use that, otherwise + # determine what the generated sources are dynamically. + generated_sources_alias = self.env.get('NINJA_GENERATED_SOURCE_ALIAS_NAME') + generated_sources_build = None + + if generated_sources_alias: + generated_sources_build = self.builds.get(generated_sources_alias) + if generated_sources_build is None or generated_sources_build["rule"] != 'phony': + raise Exception( + "ERROR: 'NINJA_GENERATED_SOURCE_ALIAS_NAME' set, but no matching Alias object found." + ) + + if generated_sources_alias and generated_sources_build: + generated_source_files = sorted( + [] if not generated_sources_build else generated_sources_build['implicit'] ) + def check_generated_source_deps(build): + return ( + build != generated_sources_build + and set(build["outputs"]).isdisjoint(generated_source_files) + ) + else: + generated_sources_build = None + generated_source_files = sorted({ + output + # First find builds which have header files in their outputs. + for build in self.builds.values() + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + }) + + if generated_source_files: + generated_sources_alias = "_ninja_generated_sources" + ninja.build( + outputs=generated_sources_alias, + rule="phony", + implicit=generated_source_files + ) + def check_generated_source_deps(build): + return ( + not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) + ) template_builders = [] scons_compiledb = False @@ -402,9 +432,7 @@ class NinjaState: # cycle. if ( generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + and check_generated_source_deps(build) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -413,7 +441,7 @@ class NinjaState: # sure that all of these sources are generated before # other builds. order_only = build.get("order_only", []) - order_only.append("_generated_sources") + order_only.append(generated_sources_alias) build["order_only"] = order_only if "order_only" in build: build["order_only"].sort() diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 3c72ee4..3dde0fa 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -276,8 +276,8 @@ def generate(env): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - # some of these construction vars could be generators, e.g. - # CommandGeneratorAction, so if the var is not a string, we + # some of these construction vars could be generators, e.g. + # CommandGeneratorAction, so if the var is not a string, we # can't parse the generated string. if isinstance(env.get(var), str): env.NinjaRuleMapping(env.get(var, None), provider) @@ -297,7 +297,7 @@ def generate(env): # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. # TODO: switch to using SCons to help determine this (Github Issue #3624) - env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = env.get('NINJA_GENERATED_SOURCE_SUFFIXES', [".h", ".hpp"]) # Force ARCOM so use 's' flag on ar instead of separately running ranlib ninja_hack_arcom(env) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 51ff435..9bf4e10 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -65,6 +65,7 @@ See its __doc__ string for a discussion of the format. NINJA_ENV_VAR_CACHE NINJA_FILE_NAME NINJA_GENERATED_SOURCE_SUFFIXES + NINJA_GENERATED_SOURCE_ALIAS_NAME NINJA_MSVC_DEPS_PREFIX NINJA_POOL NINJA_REGENERATE_DEPS @@ -198,6 +199,17 @@ python -m pip install ninja + + + + A string matching the name of a user defined alias which represents a list of all generated sources. + This will prevent the auto-detection of generated sources from &cv-NINJA_GENERATED_SOURCE_SUFFIXES;. + Then all other source files will be made to depend on this in the &ninja; build file, forcing the + generated sources to be built first. + + + + @@ -262,7 +274,7 @@ python -m pip install ninja - If not explicitly set, &SCons; will generate this dynamically from the + If not explicitly set, &SCons; will generate this dynamically from the execution environment stored in the current &consenv; (e.g. env['ENV']) where those values differ from the existing shell.. diff --git a/test/ninja/generated_sources_alias.py b/test/ninja/generated_sources_alias.py new file mode 100644 index 0000000..2c4ed36 --- /dev/null +++ b/test/ninja/generated_sources_alias.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_generated_sources_alias', 'SConstruct') + +for alias_value in [0, 1]: + + # Gen with source alias + test.run(arguments=['--disable-execute-ninja', f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None) + test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) + test.must_not_exist(test.workpath('gen_source' + _exe)) + + # run ninja independently + program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin + test.run(program=program, stdout=None) + test.run(program=test.workpath('gen_source' + _exe), stdout="4") + + # generate simple build + test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None) + test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) + test.must_contain_all(test.stdout(), 'Executing:') + test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) + test.run(program=test.workpath('gen_source' + _exe), stdout="4") + + # clean build and ninja files + test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}', '-c'], stdout=None) + test.must_contain_all_lines(test.stdout(), [ + 'Removed generated_header1.htest', + 'Removed generated_header2.htest', + 'Removed generated_header2.c', + 'Removed build.ninja']) + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/gen_source.c b/test/ninja/ninja-fixture/gen_source.c new file mode 100644 index 0000000..38ac3fe --- /dev/null +++ b/test/ninja/ninja-fixture/gen_source.c @@ -0,0 +1,9 @@ +#include +#include + +#include "generated_header2.htest" + +int main(int argc, char *argv[]) +{ + printf("%d", func2()); +} diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias new file mode 100644 index 0000000..3661bcc --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias @@ -0,0 +1,41 @@ +import SCons + +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +vars = Variables() +vars.Add("USE_GEN_SOURCE_ALIAS") + +env = Environment(variables=vars) +my_gen_alias = 'my_ninja_gen_sources' + + +env['SCANNERS'][0].add_scanner('.htest', SCons.Tool.CScanner) +env.Command('out.txt', [], f'echo USE_GEN_SOURCE_ALIAS={env["USE_GEN_SOURCE_ALIAS"]}') + +if env['USE_GEN_SOURCE_ALIAS'] == "1": + env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = my_gen_alias +else: + env['NINJA_GENERATED_SOURCE_SUFFIXES'] = ['.htest'] + +env.Tool('ninja') + +env.Alias(my_gen_alias, env.Textfile('generated_header1.htest', [ + '#pragma once', + 'int func1(){return 4;};' +])) +alias = env.Alias(my_gen_alias, env.Textfile('generated_header2.htest', [ + '#pragma once', + '', + 'int func2();' +])) +env.Depends(alias, 'out.txt') + +my_gen_alias, env.Textfile('generated_header2.c', [ + '#include "generated_header1.htest"', + '#include "generated_header2.htest"', + '', + 'int func2(){return func1();}' +]) + +env.Program(target='gen_source', source=['gen_source.c', 'generated_header2.c']) \ No newline at end of file -- cgit v0.12 From fbb04473b675db5c561f02986ec406519a194d58 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 Apr 2022 15:52:51 -0500 Subject: update changes and releases text files --- CHANGES.txt | 6 ++++-- RELEASE.txt | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8390f00..ca0756a 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,12 +18,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER will include what is the minimum supported version of Python - Fix ActionTests to work with python 3.10.1 (and higher) NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) - Action._subproc() can now be used as a python context manager to ensure that the - POpen object is properly closed. + POpen object is properly closed. (Thanks to Mats Wichmann for catching that DummyPopen needed additional logic) - Added project_url for mailing lists and Discord - Updated project url in steup.cfg to be https instead of http @@ -44,6 +44,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Added contribution from MongoDB ninja alias generated source tagging: + https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index be2444e..fe3d254 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -7,7 +7,7 @@ on the SCons download page: Here is a summary of the changes since 4.3.1: NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. @@ -58,6 +58,8 @@ FIXES with no defaults, so old usage where _defines() was called without source and target arguments would yield an exception. This issue was found via qt4 and qt5 tools in scons-contrib https://github.com/SCons/scons-contrib/issues/45 +- Added contribution from MongoDB ninja alias generated source tagging: + https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f IMPROVEMENTS ------------ -- cgit v0.12 From f75ca45685c3a026108bcd1163cd15b645b00a05 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 14 Apr 2022 19:36:37 -0700 Subject: [ci skip] Updated wording in CHANGES.txt and RELEASE.txt to more fully describe the failure which the PR fixes. --- CHANGES.txt | 5 ++++- RELEASE.txt | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4521fb9..510e0e6 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,7 +12,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Joseph Brill: - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception when the user specified msvc script does not exist. - - Fix the mingw tool so that the generate and exists methods use the same mingw search paths + - Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and + did not explicitly request the mingw tool, mingw tool initialization would fail and set the + default compiler to MSVC which wasn't installed, yielding broken build. + Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). From William Deegan: diff --git a/RELEASE.txt b/RELEASE.txt index aee7491..49636bc 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -58,8 +58,11 @@ FIXES with no defaults, so old usage where _defines() was called without source and target arguments would yield an exception. This issue was found via qt4 and qt5 tools in scons-contrib https://github.com/SCons/scons-contrib/issues/45 -- Fixed an inconsistency in the mingw tool search paths which could result in a mingw - installation not being detected (issue #4134). +- Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and + did not explicitly request the mingw tool, mingw tool initialization would fail and set the + default compiler to MSVC which wasn't installed, yielding broken build. + Updated mingw tool so that the generate and exists methods use the same mingw search paths + (issue #4134). IMPROVEMENTS ------------ -- cgit v0.12 From 1857567a3e5e2838d135b13f925da8ed89f7f4ef Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 15 Apr 2022 11:33:48 -0500 Subject: more verbose changes.txt --- CHANGES.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index ca0756a..f07ec57 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -44,7 +44,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. - - Added contribution from MongoDB ninja alias generated source tagging: + - Added contribution from MongoDB ninja alias generated source tagging. This allows users + to optionally set an alias which the ninja tool will use to determine which files are + generated source, and therefore build them first as a workaround to not having + scons scanners. The downstream commit is here: https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f From Mats Wichmann: -- cgit v0.12 From 58c61f46c8d8b77f595517155eeae9eca3aceaac Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 15 Apr 2022 22:55:39 -0500 Subject: fixed sider complaints --- SCons/Tool/ninja/NinjaState.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index c9d8a89..330b485 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -376,6 +376,7 @@ class NinjaState: generated_source_files = sorted( [] if not generated_sources_build else generated_sources_build['implicit'] ) + def check_generated_source_deps(build): return ( build != generated_sources_build @@ -404,6 +405,7 @@ class NinjaState: rule="phony", implicit=generated_source_files ) + def check_generated_source_deps(build): return ( not build["rule"] == "INSTALL" -- cgit v0.12 From 2fcccedad68aa9dd4852f63fb74df2b42099460d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 18 Apr 2022 18:09:27 -0400 Subject: Change host default target map construction to use default target list for true host instead of compatible host. Define arm64 host for VS2017 and later with amd64/x86 hosts (i.e., no native arm64 support yet). --- SCons/Tool/MSCommon/vc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index fa4c413..66a69b2 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -139,6 +139,7 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host for host, host_list in all_hosts.items(): host_all_targets_map[host] = [] for host_platform in host_list: + # all_targets[host_platform]: all targets for compatible host for target in all_targets[host_platform]: for host_target in host_target_map[host_platform][target]: for host_key in (host, all): @@ -152,7 +153,8 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host for host, host_list in all_hosts.items(): host_def_targets_map[host] = [] for host_platform in host_list: - for target in def_targets[host_platform]: + # def_targets[host]: default targets for true host + for target in def_targets[host]: for host_target in host_target_map[host_platform][target]: if host_target not in host_def_targets_map[host]: host_def_targets_map[host].append(host_target) @@ -231,16 +233,19 @@ _GE2017_HOST_TARGET_CFG = _host_target_config_factory( host_all_hosts = OrderedDict([ ('amd64', ['amd64', 'x86']), ('x86', ['x86']), + ('arm64', ['amd64', 'x86']), ]), host_all_targets = { - 'amd64': ['amd64', 'x86', 'arm', 'arm64'], + 'amd64': ['amd64', 'x86', 'arm64', 'arm'], 'x86': ['x86', 'amd64', 'arm', 'arm64'], + 'arm64': [], }, host_def_targets = { 'amd64': ['amd64', 'x86'], 'x86': ['x86'], + 'arm64': ['arm64', 'arm'], }, ) -- cgit v0.12 From 94715620f2bb87438a86a9dea7b9b3d59a5c15a7 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 19 Apr 2022 05:26:27 -0400 Subject: Rename and move script error regular expression. Make all script error regular expression components raw strings. --- SCons/Tool/MSCommon/vc.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 66a69b2..fddf19a 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -101,6 +101,22 @@ _ARCH_TO_CANONICAL = { "aarch64" : "arm64", } +# The msvc batch files report errors via stdout. The following +# regular expression attempts to match known msvc error messages +# written to stdout. +re_script_output_error = re.compile( + r'^(' + r'|'.join([ + r'VSINSTALLDIR variable is not set', # 2002-2003 + r'The specified configuration type is missing', # 2005+ + r'Error in script usage', # 2005+ + r'ERROR\:', # 2005+ + r'\!ERROR\!', # 2015-2015 + r'\[ERROR\:', # 2017+ + r'\[ERROR\]', # 2017+ + r'Syntax\:', # 2017+ + ]) + r')' +) + # host/target candidate lists _HOST_TARGET_CONFIG_NT = namedtuple("HostTargetConfig", [ # defined @@ -916,35 +932,21 @@ def reset_installed_vcs(): # of time not parsing every time. script_env_cache = None -re_output_error = None def script_env(script, args=None): global script_env_cache - global re_output_error if script_env_cache is None: script_env_cache = common.read_script_env_cache() cache_key = "{}--{}".format(script, args) cache_data = script_env_cache.get(cache_key, None) if cache_data is None: - if re_output_error is None: - re_output_error = re.compile('^(' + '|'.join([ - r'VSINSTALLDIR variable is not set', # 2002-2003 - r'The specified configuration type is missing', # 2005+ - r'Error in script usage', # 2005+ - r'ERROR\:', # 2005+ - r'\!ERROR\!', # 2015-2015 - r'\[ERROR\:', # 2017+ - r'\[ERROR\]', # 2017+ - r'Syntax\:', # 2017+ - ]) + ')') - stdout = common.get_output(script, args) # Stupid batch files do not set return code: we take a look at the # beginning of the output for an error message instead olines = stdout.splitlines() - if re_output_error.match(olines[0]): + if re_script_output_error.match(olines[0]): raise BatchFileExecutionError("\n".join(olines[:2])) cache_data = common.parse_output(stdout) -- cgit v0.12 From 8b9c4ccf1cfa42d1d26987ea6a7601ac71740158 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 19 Apr 2022 08:17:51 -0400 Subject: Define arm host for VS2017 and later with x86 host. --- SCons/Tool/MSCommon/vc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index fddf19a..6b0c677 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -250,18 +250,21 @@ _GE2017_HOST_TARGET_CFG = _host_target_config_factory( ('amd64', ['amd64', 'x86']), ('x86', ['x86']), ('arm64', ['amd64', 'x86']), + ('arm', ['x86']), ]), host_all_targets = { 'amd64': ['amd64', 'x86', 'arm64', 'arm'], 'x86': ['x86', 'amd64', 'arm', 'arm64'], 'arm64': [], + 'arm': [], }, host_def_targets = { 'amd64': ['amd64', 'x86'], 'x86': ['x86'], 'arm64': ['arm64', 'arm'], + 'arm': ['arm'], }, ) -- cgit v0.12 From 0d7f0fd6a645450acfe84ce45ff7df4680da948f Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 19 Apr 2022 09:10:34 -0400 Subject: Add comment describing host/target configuration data structure. --- SCons/Tool/MSCommon/vc.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 6b0c677..7789bca 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -117,7 +117,51 @@ re_script_output_error = re.compile( ]) + r')' ) -# host/target candidate lists +# Lists of compatible host/target combinations are derived from a set of defined +# constant data structures for each host architecture. The derived data structures +# implicitly handle the differences in full versions and express versions of visual +# studio. The host/target combination search lists are contructed in order of +# preference. The construction of the derived data structures is independent of actual +# visual studio installations. The host/target configurations are used in both the +# initial msvc detection and when finding a valid batch file for a given host/target +# combination. +# +# HostTargetConfig description: +# +# label: +# Name used for identification. +# +# host_all_hosts: +# Defined list of compatible architectures for each host architecture. +# +# host_all_targets: +# Defined list of target architectures for each host architecture. +# +# host_def_targets: +# Defined list of default target architectures for each host architecture. +# +# all_pairs: +# Derived list of all host/target combination tuples. +# +# host_target_map: +# Derived list of all compatible host/target combinations for each +# supported host/target combination. +# +# host_all_targets_map: +# Derived list of all compatible host/target combinations for each +# supported host. This is used in the initial check that cl.exe exists +# in the requisite visual studio vc host/target directory for a given host. +# +# host_def_targets_map: +# Derived list of default compatible host/target combinations for each +# supported host. This is used for a given host when the user does not +# request a target archicture. +# +# target_host_map: +# Derived list of compatible host/target combinations for each supported +# target/host combination. This is used for a given host and target when +# the user requests a target architecture. + _HOST_TARGET_CONFIG_NT = namedtuple("HostTargetConfig", [ # defined "label", # name for debugging/output -- cgit v0.12 From 0ef7e3d5bcfd7e6feb7b0277b8084e2e17e57040 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 19 Apr 2022 07:25:39 -0600 Subject: Uguide: fix a typo of environment [ci skip] Spotted by @night-ripper on Discord Signed-off-by: Mats Wichmann --- doc/user/environments.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/environments.xml b/doc/user/environments.xml index 03dc89e..7118f21 100644 --- a/doc/user/environments.xml +++ b/doc/user/environments.xml @@ -1618,7 +1618,7 @@ env.PrependUnique(CCFLAGS=['-g']) - Rather than creating a cloned environmant for specific tasks, + Rather than creating a cloned &consenv; for specific tasks, you can override or add construction variables when calling a builder method by passing them as keyword arguments. The values of these overridden or added variables will only be in -- cgit v0.12 From a941117efb1d88c05f8e2ba1175af55935523ed2 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 19 Apr 2022 09:35:44 -0400 Subject: Add blurbs to CHANGES.txt and RELEASE.txt. --- CHANGES.txt | 9 ++++++++- RELEASE.txt | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 510e0e6..24f68b5 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -13,10 +13,17 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception when the user specified msvc script does not exist. - Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and - did not explicitly request the mingw tool, mingw tool initialization would fail and set the + did not explicitly request the mingw tool, mingw tool initialization would fail and set the default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). + - Refactor the data definitions for msvc configurations to allow derived data structures to be + constructed during initialization that removes the need for special case handling during runtime + execution. Special case handling of host/target combinations is eliminated and replaced with + pre-computed search lists that implicitly handle the differences between full versions and + express versions of msvc. The refactoring is motivated by issues arising from the differences + between initial msvc detection and msvc batch file determination when configuring the build + environment (issues #2668 and #2697). From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message diff --git a/RELEASE.txt b/RELEASE.txt index 49636bc..091ba96 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -59,10 +59,14 @@ FIXES arguments would yield an exception. This issue was found via qt4 and qt5 tools in scons-contrib https://github.com/SCons/scons-contrib/issues/45 - Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and - did not explicitly request the mingw tool, mingw tool initialization would fail and set the + did not explicitly request the mingw tool, mingw tool initialization would fail and set the default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). +- Fix issue where Express versions of the MSVC compiler were not detected due to differences + in initial msvc detection and msvc batch file determination when configuring the build + environment. This could lead to build failures when only an MSVC Express instance is installed + and the MSVC version is not explicitly specified (issue #2668 and issue #2697). IMPROVEMENTS ------------ @@ -73,6 +77,10 @@ IMPROVEMENTS - Added ninja mingw support and improved ninja CommandGeneratorAction support. - Command-line help is now sensitive to the size of the terminal window: the width of the help text will scale for terminals other than 80 chars wide. +- Refactor the msvc code so that the same data structures are used during initial msvc detection + and msvc batch file determination when configuring the build environment. Simplify the msvc + code by eliminating special case handling primarily due to the differences between the full + versions and express versions of visual studio. PACKAGING --------- -- cgit v0.12 From bee592f06bef6018072258db5f688d8da6b1652e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 16 Apr 2022 01:17:25 -0500 Subject: added workarounds for ninja with python36 on windows --- SCons/Tool/ninja/ninja_run_daemon.py | 18 ++++++++++++++---- SCons/Tool/ninja/ninja_scons_daemon.py | 7 +++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 057537a..9f71bd1 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -67,10 +67,20 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): ] + sys.argv[1:] logging.debug(f"Starting daemon with {' '.join(cmd)}") - p = subprocess.Popen( - cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False - ) - + + if sys.platform == 'win32' and sys.version_info[0] == 3 and sys.version_info[1] == 6: + # on windows with python version 3.6, popen does not do a good job disconnecting + # the std handles and this make ninja hang becuase they stay open to the original + # process ninja launched. Here we can force the handles to be separated. + si = subprocess.STARTUPINFO() + si.dwFlags = subprocess.STARTF_USESTDHANDLES + p = subprocess.Popen( + cmd, close_fds=True, shell=False, startupinfo=si + ) + else: + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False, + ) with open(daemon_dir / "pidfile", "w") as f: f.write(str(p.pid)) with open(ninja_builddir / "scons_daemon_dirty", "w") as f: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 88f3b22..9c51731 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -56,6 +56,13 @@ ninja_builddir = pathlib.Path(sys.argv[2]) daemon_keep_alive = int(sys.argv[3]) args = sys.argv[4:] +# windows and python36 passed nothing for the std handles because of issues with popen +# and its handles so we have to make some fake ones to prevent exceptions. +if sys.platform == 'win32' and sys.version_info[0] == 3 and sys.version_info[1] == 6: + from io import StringIO + sys.stderr = StringIO() + sys.stdout = StringIO() + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) ) -- cgit v0.12 From c75902a661af78f16b8f3a2c8c39df7927a6ef34 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 16 Apr 2022 01:22:23 -0500 Subject: fix spelling sider issue --- SCons/Tool/ninja/ninja_run_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 9f71bd1..1e767dc 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -70,7 +70,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): if sys.platform == 'win32' and sys.version_info[0] == 3 and sys.version_info[1] == 6: # on windows with python version 3.6, popen does not do a good job disconnecting - # the std handles and this make ninja hang becuase they stay open to the original + # the std handles and this make ninja hang because they stay open to the original # process ninja launched. Here we can force the handles to be separated. si = subprocess.STARTUPINFO() si.dwFlags = subprocess.STARTF_USESTDHANDLES -- cgit v0.12 From bc71cda030eed0f183271d1973ead66a7c7b075a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 20 Apr 2022 12:35:33 -0500 Subject: update changes/releases texts --- CHANGES.txt | 3 +++ RELEASE.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 510e0e6..19b35cc 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -49,6 +49,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Added special case for ninja scons daemon to work in win32 python3.6 environments. + This particular environment does a bad job managing popen standard file handles, so + some special workarounds are needed. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 49636bc..85526cd 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -63,6 +63,9 @@ FIXES default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). +- Added special case for ninja scons daemon to work in win32 python3.6 environments. + This particular environment does a bad job managing popen standard file handles, so + some special workarounds are needed. IMPROVEMENTS ------------ -- cgit v0.12 From 0b436cc43a7e16bcb3aefca960bb7faa39cd8578 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 20 Apr 2022 12:45:47 -0500 Subject: Update ninja file generation to only create response files for build commands which exceed MAXLINELENGTH --- CHANGES.txt | 2 + RELEASE.txt | 2 + SCons/Tool/ninja/Methods.py | 5 +- SCons/Tool/ninja/NinjaState.py | 40 +++++++-- .../sconstruct_response_file | 15 ++++ test/ninja/response_file.py | 98 ++++++++++++++++++++++ 6 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_response_file create mode 100644 test/ninja/response_file.py diff --git a/CHANGES.txt b/CHANGES.txt index 510e0e6..17bd39d 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -49,6 +49,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Update ninja file generation to only create response files for build commands + which exceed MAXLINELENGTH From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 49636bc..1a8e916 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -42,6 +42,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - The change to "content" and "content-timestamp" Decider names is reflected in the User Guide as well, since the hash function may be other than md5 (tidying up from earlier change) +- Update ninja file generation to only create response files for build commands + which exceed MAXLINELENGTH FIXES diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 4c6aa2f..2be576f 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -276,7 +276,10 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): ret_rule = 'TEMPLATE' else: - ret_rule = rule + if len(' '.join(cmd_list)) < env.get('MAXLINELENGTH', 2048): + ret_rule = rule + else: + ret_rule = rule + '_RSP' return ret_rule, variables, [tool_command] diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index fff60be..46df29a 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -132,19 +132,19 @@ class NinjaState: # use this to generate a compile_commands.json database # which can't use the shell command as it's compile # command. - "CC": { + "CC_RSP": { "command": "$env$CC @$out.rsp", "description": "Compiling $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, - "CXX": { + "CXX_RSP": { "command": "$env$CXX @$out.rsp", "description": "Compiling $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, - "LINK": { + "LINK_RSP": { "command": "$env$LINK @$out.rsp", "description": "Linking $out", "rspfile": "$out.rsp", @@ -157,7 +157,7 @@ class NinjaState: # Native SCons will perform this operation so we need to force ninja # to do the same. See related for more info: # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { + "AR_RSP": { "command": "{}$env$AR @$out.rsp".format( '' if sys.platform == "win32" else "rm -f $out && " ), @@ -166,6 +166,26 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, + "CC": { + "command": "$env$CC $rspc", + "description": "Compiling $out", + }, + "CXX": { + "command": "$env$CXX $rspc", + "description": "Compiling $out", + }, + "LINK": { + "command": "$env$LINK $rspc", + "description": "Linking $out", + "pool": "local_pool", + }, + "AR": { + "command": "{}$env$AR $rspc".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -339,7 +359,17 @@ class NinjaState: if self.__generated: return - self.rules.update(self.env.get(NINJA_RULES, {})) + + for key, rule in self.env.get(NINJA_RULES, {}).items(): + # make a non response file rule for users custom response file rules. + if rule.get('rspfile') is not None: + self.rules.update({key + '_RSP': rule}) + non_rsp_rule = rule.copy() + del non_rsp_rule['rspfile'] + del non_rsp_rule['rspfile_content'] + self.rules.update({key: non_rsp_rule}) + else: + self.rules.update({key: rule}) self.pools.update(self.env.get(NINJA_POOLS, {})) content = io.StringIO() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_response_file b/test/ninja/ninja_test_sconscripts/sconstruct_response_file new file mode 100644 index 0000000..6b7eb5d --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_response_file @@ -0,0 +1,15 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +vars = Variables() +vars.Add('MLL', default='2048') + +env = Environment(variables=vars) +env['MAXLINELENGTH'] = int(env['MLL']) +env.Tool('ninja') + +env.Program(target='foo', source='foo.c', OBJSUFFIX=env['OBJSUFFIX'] + "1") + +env2 = env.Clone() +env2.Append(CPPPATH='very/long/and/very/fake/path/for/testing') +env2.Program(target='foo2', source='foo.c', OBJSUFFIX=env['OBJSUFFIX'] + "2") \ No newline at end of file diff --git a/test/ninja/response_file.py b/test/ninja/response_file.py new file mode 100644 index 0000000..5976e05 --- /dev/null +++ b/test/ninja/response_file.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import sys +import json +from io import StringIO + + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe +_obj = TestSCons._obj + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_response_file', 'SConstruct') + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin + +# now we must extract the line length for a given command by print the command +# to a StringIO internal buffer and calculating the length of the resulting command. +test.run(program=[program, 'compiledb'], stdout=None) +command_length = None +with open('compile_commands.json') as f: + comp_commands = json.loads(f.read()) + for cmd in comp_commands: + if cmd['output'].endswith('1'): + result = StringIO() + old_stdout = sys.stdout + sys.stdout = result + print(cmd['command']) + result_string = result.getvalue() + sys.stdout = old_stdout + if sys.platform == 'win32': + command_length = len(result_string) + else: + command_length = len(result_string.split(';')[-1]) + +# now regen the ninja file with the MAXLINELENGTH so we can force the response file in one case. +test.run(arguments=['--disable-execute-ninja', 'MLL='+str(command_length)], stdout=None) +test.run(program=[program, '-v', '-d', 'keeprsp'], stdout=None) +test.must_not_exist(test.workpath('foo' + _obj + '1.rsp')) +test.must_exist(test.workpath('foo' + _obj + '2.rsp')) + +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +test.run(program=program, arguments='exit_ninja_scons_daemon', stdout=None) +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From acd04eeceb5e2d43eb0c9a6ccb3afd79e62f82be Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 20 Apr 2022 15:34:18 -0500 Subject: remove test clean up call --- test/ninja/response_file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ninja/response_file.py b/test/ninja/response_file.py index 5976e05..3d23c2b 100644 --- a/test/ninja/response_file.py +++ b/test/ninja/response_file.py @@ -88,7 +88,6 @@ test.must_exist(test.workpath('foo' + _obj + '2.rsp')) test.run(program=test.workpath('foo' + _exe), stdout="foo.c") -test.run(program=program, arguments='exit_ninja_scons_daemon', stdout=None) test.pass_test() # Local Variables: -- cgit v0.12 From d662abf5ae685d31348d63cc1ec6b571d8a69c1b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 Apr 2022 11:35:30 -0500 Subject: added mingw depfile format --- SCons/Tool/ninja/NinjaState.py | 6 ++++-- SCons/Tool/ninja/__init__.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index fff60be..6926ba3 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -263,11 +263,13 @@ class NinjaState: } for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": + if env["NINJA_DEPFILE_PARSE_FORMAT"] == "msvc": self.rules[rule]["deps"] = "msvc" - else: + elif env["NINJA_DEPFILE_PARSE_FORMAT"] == "gcc" or env["NINJA_DEPFILE_PARSE_FORMAT"] == "clang": self.rules[rule]["deps"] = "gcc" self.rules[rule]["depfile"] = "$out.d" + else: + raise Exception(f"Unknown 'NINJA_DEPFILE_PARSE_FORMAT'={env['NINJA_DEPFILE_PARSE_FORMAT']}, use 'mvsc', 'gcc', or 'clang'.") def add_build(self, node): if not node.has_builder(): diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 3c72ee4..affe5d9 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -226,6 +226,8 @@ def generate(env): pass else: env.Append(CCFLAGS='$CCDEPFLAGS') + + env["NINJA_DEPFILE_PARSE_FORMAT"] = env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if env['PLATFORM'] == 'win32' else 'gcc') env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") -- cgit v0.12 From 69da4a816d6e39bf3d816bc39d949a44164de677 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 Apr 2022 13:38:34 -0500 Subject: updated tools to set this --- CHANGES.txt | 2 ++ RELEASE.txt | 2 ++ SCons/Tool/clang.py | 2 +- SCons/Tool/clangxx.py | 1 + SCons/Tool/gcc.py | 1 + SCons/Tool/gxx.py | 1 + SCons/Tool/msvc.py | 1 + SCons/Tool/ninja/ninja.xml | 12 ++++++++++++ 8 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 510e0e6..de0c01e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -49,6 +49,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Added user configurable setting of ninja depfile format. This helps in mingw environments + where the gcc/clang format might be used on windows. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 49636bc..c4e0714 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -63,6 +63,8 @@ FIXES default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). +- Added user configurable setting of ninja depfile format. This helps in mingw environments + where the gcc/clang format might be used on windows. IMPROVEMENTS ------------ diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 2a12a31..518b09e 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -82,7 +82,7 @@ def generate(env): env['CCVERSION'] = match.group(1) env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' - + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'clang' def exists(env): diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index a78dc6c..07d8378 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -90,6 +90,7 @@ def generate(env): env['CXXVERSION'] = match.group(1) env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'clang' def exists(env): diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 94dfad3..1a25cb4 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -58,6 +58,7 @@ def generate(env): env['CCVERSION'] = version env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index cc93f93..1272997 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -65,6 +65,7 @@ def generate(env): env['CXXVERSION'] = version env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' diff --git a/SCons/Tool/msvc.py b/SCons/Tool/msvc.py index f2cd418..58b7463 100644 --- a/SCons/Tool/msvc.py +++ b/SCons/Tool/msvc.py @@ -316,6 +316,7 @@ def generate(env): env['ENV']['SystemRoot'] = SCons.Platform.win32.get_system_root() env['CCDEPFLAGS'] = '/showIncludes' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'msvc' def exists(env): diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 51ff435..b875922 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -66,6 +66,7 @@ See its __doc__ string for a discussion of the format. NINJA_FILE_NAME NINJA_GENERATED_SOURCE_SUFFIXES NINJA_MSVC_DEPS_PREFIX + NINJA_DEPFILE_PARSE_FORMAT NINJA_POOL NINJA_REGENERATE_DEPS NINJA_SYNTAX @@ -209,6 +210,17 @@ python -m pip install ninja + + + + Determines the type of format ninja should expect when parsing header + include depfiles. Can be , , or . + The option corresponds to format, and + or correspond to . + + + + -- cgit v0.12 From 121ed84c0dfe5d9d21655b7fc4fce0a57db346a4 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 Apr 2022 23:44:08 -0500 Subject: check the value at a later time --- SCons/Tool/ninja/NinjaState.py | 5 +++-- SCons/Tool/ninja/__init__.py | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 6926ba3..cd1de7e 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -262,10 +262,11 @@ class NinjaState: "scons_pool": 1, } + deps_format = env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if env['PLATFORM'] == 'win32' else 'gcc') for rule in ["CC", "CXX"]: - if env["NINJA_DEPFILE_PARSE_FORMAT"] == "msvc": + if deps_format == "msvc": self.rules[rule]["deps"] = "msvc" - elif env["NINJA_DEPFILE_PARSE_FORMAT"] == "gcc" or env["NINJA_DEPFILE_PARSE_FORMAT"] == "clang": + elif deps_format == "gcc" or deps_format == "clang": self.rules[rule]["deps"] = "gcc" self.rules[rule]["depfile"] = "$out.d" else: diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index affe5d9..4820ee1 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -227,8 +227,6 @@ def generate(env): else: env.Append(CCFLAGS='$CCDEPFLAGS') - env["NINJA_DEPFILE_PARSE_FORMAT"] = env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if env['PLATFORM'] == 'win32' else 'gcc') - env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command -- cgit v0.12 From 22c23dc123f5c39fff0912c9b099487f6d2b0522 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 Apr 2022 23:49:36 -0500 Subject: delayed setting till generation --- SCons/Tool/ninja/NinjaState.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index cd1de7e..034a02e 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -254,23 +254,7 @@ class NinjaState: "description": "Archiving $out", "pool": "local_pool", } - - num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) - self.pools = { - "local_pool": num_jobs, - "install_pool": num_jobs / 2, - "scons_pool": 1, - } - - deps_format = env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if env['PLATFORM'] == 'win32' else 'gcc') - for rule in ["CC", "CXX"]: - if deps_format == "msvc": - self.rules[rule]["deps"] = "msvc" - elif deps_format == "gcc" or deps_format == "clang": - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" - else: - raise Exception(f"Unknown 'NINJA_DEPFILE_PARSE_FORMAT'={env['NINJA_DEPFILE_PARSE_FORMAT']}, use 'mvsc', 'gcc', or 'clang'.") + self.pools = {"scons_pool": 1} def add_build(self, node): if not node.has_builder(): @@ -342,6 +326,22 @@ class NinjaState: if self.__generated: return + num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) + self.pools.update({ + "local_pool": num_jobs, + "install_pool": num_jobs / 2, + }) + + deps_format = self.env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if self.env['PLATFORM'] == 'win32' else 'gcc') + for rule in ["CC", "CXX"]: + if deps_format == "msvc": + self.rules[rule]["deps"] = "msvc" + elif deps_format == "gcc" or deps_format == "clang": + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" + else: + raise Exception(f"Unknown 'NINJA_DEPFILE_PARSE_FORMAT'={self.env['NINJA_DEPFILE_PARSE_FORMAT']}, use 'mvsc', 'gcc', or 'clang'.") + self.rules.update(self.env.get(NINJA_RULES, {})) self.pools.update(self.env.get(NINJA_POOLS, {})) -- cgit v0.12 From f2c025d668b9d0b09f75a9d79020c1a987bf1f11 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 30 Apr 2022 12:46:44 -0400 Subject: Use the logging framework for mscommon debug output to stdout --- CHANGES.txt | 3 +++ SCons/Tool/MSCommon/common.py | 36 ++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 510e0e6..415212b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). + - Update the internal mscommon debug handling for stdout to use the logging framework. Update + the debug record format so that the output logged to stdout contains the output that is logged + to a file. From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index ee31b2d..c5cf232 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -37,14 +37,7 @@ import SCons.Util # SCONS_MSCOMMON_DEBUG is internal-use so undocumented: # set to '-' to print to console, else set to filename to log to LOGFILE = os.environ.get('SCONS_MSCOMMON_DEBUG') -if LOGFILE == '-': - def debug(message, *args): - if args: - print(message % args) - else: - print(message) - -elif LOGFILE: +if LOGFILE: import logging modulelist = ( # root module and parent/root module @@ -71,17 +64,24 @@ elif LOGFILE: relfilename = relfilename.replace('\\', '/') record.relfilename = relfilename return True + # Log format looks like: + # 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [file] + # debug:00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [stdout] + log_format=( + '%(relativeCreated)05dms' + ':%(relfilename)s' + ':%(funcName)s' + '#%(lineno)s' + ': %(message)s' + ) + if LOGFILE == '-': + log_format = 'debug:' + log_format + log_handler = logging.StreamHandler(sys.stdout) + else: + log_handler = logging.FileHandler(filename=LOGFILE) logging.basicConfig( - # This looks like: - # 00109ms:MSCommon/vc.py:find_vc_pdir#447:VC found '14.3' - format=( - '%(relativeCreated)05dms' - ':%(relfilename)s' - ':%(funcName)s' - '#%(lineno)s' - ':%(message)s' - ), - filename=LOGFILE, + format=log_format, + handlers=[log_handler], level=logging.DEBUG) logger = logging.getLogger(name=__name__) logger.addFilter(_Debug_Filter()) -- cgit v0.12 From e4545999ed50c1df21f0d2daab48f0fbfbe19259 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 30 Apr 2022 17:16:02 -0400 Subject: Add space to stdout debug prefix for logging record. --- SCons/Tool/MSCommon/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index c5cf232..c3ab6ad 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -75,7 +75,7 @@ if LOGFILE: ': %(message)s' ) if LOGFILE == '-': - log_format = 'debug:' + log_format + log_format = 'debug: ' + log_format log_handler = logging.StreamHandler(sys.stdout) else: log_handler = logging.FileHandler(filename=LOGFILE) -- cgit v0.12 From 03d49b630465628815ab4d24eedd8b5a7689e6d7 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 30 Apr 2022 17:46:29 -0400 Subject: Update debug record format comment for additional space. --- SCons/Tool/MSCommon/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index c3ab6ad..83bd2fd 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -65,8 +65,8 @@ if LOGFILE: record.relfilename = relfilename return True # Log format looks like: - # 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [file] - # debug:00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [stdout] + # 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [file] + # debug: 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [stdout] log_format=( '%(relativeCreated)05dms' ':%(relfilename)s' -- cgit v0.12 From 0aca2c9e7f11f29d21624ede1799f9745c68ef1e Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 3 May 2022 10:09:44 -0400 Subject: Update CHANGES.txt to explicitly reference fix motivating change [ci skip] --- CHANGES.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 24f68b5..4bf023e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,12 +18,15 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). - Refactor the data definitions for msvc configurations to allow derived data structures to be - constructed during initialization that removes the need for special case handling during runtime - execution. Special case handling of host/target combinations is eliminated and replaced with - pre-computed search lists that implicitly handle the differences between full versions and - express versions of msvc. The refactoring is motivated by issues arising from the differences - between initial msvc detection and msvc batch file determination when configuring the build - environment (issues #2668 and #2697). + constructed during initialization that removes the need for special case handling during + runtime execution. Special case handling of host/target combinations is eliminated and + replaced with pre-computed search lists that implicitly handle the differences between full + versions and express versions of msvc. This fixes an issue where Express versions of the MSVC + compiler were not detected due to differences in initial msvc detection and msvc batch file + determination when configuring the build environment. This could lead to build failures when + only an MSVC Express instance is installed and the MSVC version is not explicitly specified + (issue #2668 and issue #2697). + From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message -- cgit v0.12 From 2083d7f530ba95caa0aa5a95ec4702cc5a10aea6 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 3 May 2022 10:30:33 -0400 Subject: Update CHANGES.txt and add same to RELEASE.txt. [ci skip] --- CHANGES.txt | 8 +++++--- RELEASE.txt | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 415212b..6f518f0 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,9 +17,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). - - Update the internal mscommon debug handling for stdout to use the logging framework. Update - the debug record format so that the output logged to stdout contains the output that is logged - to a file. + - Update the debug output written to stdout for MSVC initialization which is enabled by setting + SCONS_MSCOMMON_DEBUG=- to use the logging module. Also changed the debug output format + written to stdout to include more information about the source for each message of MSVC + initialization debugging output. A single space was added before the message for all + debugging output records written to stdout and to files. From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message diff --git a/RELEASE.txt b/RELEASE.txt index 49636bc..a4f15d9 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -42,6 +42,11 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - The change to "content" and "content-timestamp" Decider names is reflected in the User Guide as well, since the hash function may be other than md5 (tidying up from earlier change) +- Update the debug output written to stdout for MSVC initialization which is enabled + by setting SCONS_MSCOMMON_DEBUG=- to use the logging module. Also changed the debug + output format written to stdout to include more information about the source for each + message of MSVC initialization debugging output. A single space was added before the + message for all debugging output records written to stdout and to files. FIXES -- cgit v0.12 From f4941654584f3bab88b7f2ee784bdc6e7eb3fdde Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 3 May 2022 21:40:44 -0500 Subject: adding test --- CHANGES.txt | 9 +-- RELEASE.txt | 5 +- test/ninja/mingw_depfile_format.py | 82 ++++++++++++++++++++++ .../sconstruct_mingw_depfile_format | 10 +++ 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 test/ninja/mingw_depfile_format.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format diff --git a/CHANGES.txt b/CHANGES.txt index d8cbc7f..0619bc2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -37,12 +37,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER will include what is the minimum supported version of Python - Fix ActionTests to work with python 3.10.1 (and higher) NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) - Action._subproc() can now be used as a python context manager to ensure that the - POpen object is properly closed. + POpen object is properly closed. (Thanks to Mats Wichmann for catching that DummyPopen needed additional logic) - Added project_url for mailing lists and Discord - Updated project url in steup.cfg to be https instead of http @@ -63,8 +63,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. - - Added user configurable setting of ninja depfile format. This helps in mingw environments - where the gcc/clang format might be used on windows. + - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected + format. Compiler tools will also configure the variable automatically. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 38f5deb..ecd1ea0 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -7,7 +7,7 @@ on the SCons download page: Here is a summary of the changes since 4.3.1: NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. @@ -47,6 +47,9 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY output format written to stdout to include more information about the source for each message of MSVC initialization debugging output. A single space was added before the message for all debugging output records written to stdout and to files. +- Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected + format. Compiler tools will also configure the variable automatically. FIXES diff --git a/test/ninja/mingw_depfile_format.py b/test/ninja/mingw_depfile_format.py new file mode 100644 index 0000000..1fdbc87 --- /dev/null +++ b/test/ninja/mingw_depfile_format.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +from pathlib import Path +import sys + +import TestSCons +from TestCmd import IS_WINDOWS +import SCons +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS + +test = TestSCons.TestSCons() + +if sys.platform not in ('cygwin', 'win32',): + test.skip_test("Skipping mingw test on non-Windows platform %s." % sys.platform) + +dp = MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS +gcc = SCons.Tool.find_program_path(test.Environment(), 'gcc', default_paths=dp) +if not gcc: + test.skip_test("Skipping mingw test, no MinGW found.\n") + +# ninja must have the os environment setup to work properly +os.environ["PATH"] += os.pathsep + os.path.dirname(gcc) + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_mingw_depfile_format', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) + +Path(test.workpath('test_header.h')).touch() + +# only generate the ninja file +test.run(stdout=None) +print(test.stdout()) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format new file mode 100644 index 0000000..d0420a8 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format @@ -0,0 +1,10 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=['mingw', 'textfile']) +env.Tool('ninja') + +env.Textfile('main.c', ['#include "test_header.h', 'int main(){return header_func();}']) +env.Textfile('test_header.h', ['#pragma once', 'int header_func(){return 0;}']) + +env.Program('out', 'main.c') \ No newline at end of file -- cgit v0.12 From 7c483cfa2e0c81aa5d7d634e758f5448a9759626 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 3 May 2022 22:43:07 -0500 Subject: fix test --- test/ninja/mingw_depfile_format.py | 10 ++++++++-- .../ninja_test_sconscripts/sconstruct_mingw_depfile_format | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/ninja/mingw_depfile_format.py b/test/ninja/mingw_depfile_format.py index 1fdbc87..39f2876 100644 --- a/test/ninja/mingw_depfile_format.py +++ b/test/ninja/mingw_depfile_format.py @@ -68,10 +68,16 @@ test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) Path(test.workpath('test_header.h')).touch() +original_mtime = Path(test.workpath('out' + _exe)).lstat().st_mtime # only generate the ninja file -test.run(stdout=None) -print(test.stdout()) +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, arguments=['-v'], stdout=None) + +new_mtime = Path(test.workpath('out' + _exe)).lstat().st_mtime + +if original_mtime == new_mtime: + test.fail_test(message="Reason: Ninja failed to rebuild the output when the header changed.") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format index d0420a8..26da918 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format +++ b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format @@ -4,7 +4,7 @@ DefaultEnvironment(tools=[]) env = Environment(tools=['mingw', 'textfile']) env.Tool('ninja') -env.Textfile('main.c', ['#include "test_header.h', 'int main(){return header_func();}']) +env.Textfile('main.c', ['#include "test_header.h"', 'int main(){return header_func();}']) env.Textfile('test_header.h', ['#pragma once', 'int header_func(){return 0;}']) env.Program('out', 'main.c') \ No newline at end of file -- cgit v0.12 From bd64d6256fb866a676435fdffeefcaed054b7e1c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 3 May 2022 22:45:25 -0500 Subject: fix release.txt --- RELEASE.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index ecd1ea0..91c86d2 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -47,10 +47,6 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY output format written to stdout to include more information about the source for each message of MSVC initialization debugging output. A single space was added before the message for all debugging output records written to stdout and to files. -- Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. - Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected - format. Compiler tools will also configure the variable automatically. - FIXES ----- @@ -71,8 +67,9 @@ FIXES default compiler to MSVC which wasn't installed, yielding broken build. Updated mingw tool so that the generate and exists methods use the same mingw search paths (issue #4134). -- Added user configurable setting of ninja depfile format. This helps in mingw environments - where the gcc/clang format might be used on windows. +- Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected + format. Compiler tools will also configure the variable automatically. - Fix issue where Express versions of the MSVC compiler were not detected due to differences in initial msvc detection and msvc batch file determination when configuring the build environment. This could lead to build failures when only an MSVC Express instance is installed -- cgit v0.12 From d0106a99676f39324c1a2053623a0c1045298450 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 27 Apr 2022 10:02:15 -0600 Subject: docs: update parseconfig, parseflags, mergeflags Docstrings Environment (inc. .backtick) also lightly updated. Functional change: the function to be called by ParseConfig is now passed the *unique* flag, previously omitted (if the default MergeFlags was used, it still picked up this flag but a user-supplied function did not). Added a unit test for user-supplied function. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + SCons/Environment.py | 75 +++++++++++++++++++--------- SCons/Environment.xml | 108 ++++++++++++++++++++--------------------- SCons/EnvironmentTests.py | 30 ++++++++++++ SCons/Tool/docbook/__init__.py | 1 + doc/user/mergeflags.xml | 36 +++++++------- doc/user/parseconfig.xml | 44 ++++++++--------- doc/user/parseflags.xml | 19 ++++---- 8 files changed, 189 insertions(+), 126 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9a6a26f..03b5d31 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -93,6 +93,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER "dict" (avoid redefining builtin) - Fix an old use-before-set bug in tex tool (issue #2888) - Fix a test harness exception returning stderr if a wait_for timed out. + - ParseConfig now correctly passes the *unique* flag to a user-supplied + flag-merging function. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Environment.py b/SCons/Environment.py index 3507263..c38b51c 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -558,20 +558,38 @@ class SubstitutionEnvironment: subst_target_source = subst - def backtick(self, command): + + def backtick(self, command) -> str: + """Emulate command substitution. + + Provides behavior conceptually like POSIX Shell notation + for running a command in backquotes (backticks) by running + ``command`` and returning the resulting output string. + + This is not really a public API any longer, it is provided for the + use of :meth:`ParseFlags` (which supports it using a syntax of + !command) and :meth:`ParseConfig`. + + Raises: + OSError: if the external command returned non-zero exit status. + """ + import subprocess + # common arguments - kw = { 'stdin' : 'devnull', - 'stdout' : subprocess.PIPE, - 'stderr' : subprocess.PIPE, - 'universal_newlines' : True, - } + kw = { + "stdin": "devnull", + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "universal_newlines": True, + } # if the command is a list, assume it's been quoted # othewise force a shell - if not is_List(command): kw['shell'] = True + if not is_List(command): + kw["shell"] = True # run constructed command p = SCons.Action._subproc(self, command, **kw) - out,err = p.communicate() + out, err = p.communicate() status = p.wait() if err: sys.stderr.write("" + err) @@ -579,6 +597,7 @@ class SubstitutionEnvironment: raise OSError("'%s' exited %d" % (command, status)) return out + def AddMethod(self, function, name=None): """ Adds the specified function as a method of this construction @@ -622,12 +641,12 @@ class SubstitutionEnvironment: env.MergeFlags(merges) return env - def ParseFlags(self, *flags): + def ParseFlags(self, *flags) -> dict: """Return a dict of parsed flags. Parse ``flags`` and return a dict with the flags distributed into the appropriate construction variable names. The flags are treated - as a typical set of command-line flags for a GNU-like toolchain, + as a typical set of command-line flags for a GNU-style toolchain, such as might have been generated by one of the {foo}-config scripts, and used to populate the entries based on knowledge embedded in this method - the choices are not expected to be portable to other @@ -815,18 +834,19 @@ class SubstitutionEnvironment: do_parse(arg) return mapping - def MergeFlags(self, args, unique=True): + def MergeFlags(self, args, unique=True) -> None: """Merge flags into construction variables. Merges the flags from ``args`` into this construction environent. - If ``args`` is not a dict, it is first converted to a dictionary with + If ``args`` is not a dict, it is first converted to one with flags distributed into appropriate construction variables. See :meth:`ParseFlags`. Args: args: flags to merge - unique: merge flags rather than appending (default: True) - + unique: merge flags rather than appending (default: True). + When merging, path variables are retained from the front, + other construction variables from the end. """ if not is_Dict(args): args = self.ParseFlags(args) @@ -1627,25 +1647,32 @@ class Base(SubstitutionEnvironment): if name[:len(prefix)] == prefix and name[-len(suffix):] == suffix: return path + def ParseConfig(self, command, function=None, unique=True): - """ - Use the specified function to parse the output of the command - in order to modify the current environment. The 'command' can - be a string or a list of strings representing a command and - its arguments. 'Function' is an optional argument that takes - the environment, the output of the command, and the unique flag. - If no function is specified, MergeFlags, which treats the output - as the result of a typical 'X-config' command (i.e. gtk-config), - will merge the output into the appropriate variables. + """Parse the result of running a command to update construction vars. + + Use ``function`` to parse the output of running ``command`` + in order to modify the current environment. + + Args: + command: a string or a list of strings representing a command + and its arguments. + function: called to process the result of ``command``, which will + be passed as ``args``. If ``function`` is omitted or ``None``, + :meth:`MergeFlags` is used. Takes 3 args ``(env, args, unique)`` + unique: whether no duplicate values are allowed (default true) """ if function is None: + def parse_conf(env, cmd, unique=unique): return env.MergeFlags(cmd, unique) + function = parse_conf if is_List(command): command = ' '.join(command) command = self.subst(command) - return function(self, self.backtick(command)) + return function(self, self.backtick(command), unique) + def ParseDepends(self, filename, must_exist=None, only_one=False): """ diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 5b41f22..3a6df97 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -2204,12 +2204,8 @@ not as separate arguments to -By default, -duplicate values are eliminated; -you can, however, specify -unique=False -to allow duplicate -values to be added. +If unique is true (the default), +duplicate values are not stored. When eliminating duplicate values, any &consvars; that end with the string @@ -2217,6 +2213,8 @@ the string keep the left-most unique value. All other &consvars; keep the right-most unique value. +If unique is false, +values are added even if they are duplicates. @@ -2233,9 +2231,13 @@ env.MergeFlags(['!pkg-config gtk+-2.0 --cflags', '-O3']) # Combine an optimization flag with the flags returned from running pkg-config # twice and merge the result into the construction variables. -env.MergeFlags(['-O3', - '!pkg-config gtk+-2.0 --cflags --libs', - '!pkg-config libpng12 --cflags --libs']) +env.MergeFlags( + [ + '-O3', + '!pkg-config gtk+-2.0 --cflags --libs', + '!pkg-config libpng12 --cflags --libs', + ] +) @@ -2347,15 +2349,13 @@ NoClean(env.Program('hello', 'hello.c')) Updates the current &consenv; with the values extracted -from the output from running external command, -by calling a helper function function -which understands -the output of command. +from the output of running external command, +by passing it to a helper function. command may be a string or a list of strings representing the command and its arguments. If function -is not given, +is omitted or None, &f-link-env-MergeFlags; is used. By default, duplicate values are not @@ -2366,33 +2366,32 @@ to allow duplicate values to be added. -If &f-env-MergeFlags; is used, -it expects a response in the style of a -*-config -command typical of the POSIX programming environment -(for example, -gtk-config) -and adds the options -to the appropriate construction variables. -Interpreted options -and the construction variables they affect -are as specified for the -&f-link-env-ParseFlags; -method (which -&f-env-MergeFlags; calls). -See that method's description -for a table of options and corresponding construction variables. +command is executed using the +SCons execution environment (that is, the &consvar; +&cv-link-ENV; in the current &consenv;). +If command needs additional information +to operate properly, that needs to be set in the execution environment. +For example, pkg-config +may need a custom value set in the PKG_CONFIG_PATH +environment variable. -If &f-env-MergeFlags; cannot interpret the results of +&f-env-MergeFlags; needs to understand +the output produced by command +in order to distribute it to appropriate &consvars;. +&f-env-MergeFlags; uses a separate function to +do that processing - +see &f-link-env-ParseFlags; for the details, including a +a table of options and corresponding construction variables. +To provide alternative processing of the output of command, you can suppply a custom -function to do so. -function -must accept three arguments: -the &consenv; to modify, the string returned -by running command, +function, +which must accept three arguments: +the &consenv; to modify, +a string argument containing the output from running +command, and the optional unique flag. @@ -2405,8 +2404,7 @@ and the optional -Parses the contents of the specified -filename +Parses the contents of filename as a list of dependencies in the style of &Make; or @@ -2417,27 +2415,21 @@ and explicitly establishes all of the listed dependencies. By default, it is not an error -if the specified -filename +if filename does not exist. The optional must_exist -argument may be set to a non-zero -value to have -scons -throw an exception and -generate an error if the file does not exist, +argument may be set to True +to have &SCons; +raise an exception if the file does not exist, or is otherwise inaccessible. The optional only_one -argument may be set to a non-zero -value to have -scons -thrown an exception and -generate an error +argument may be set to True +to have &SCons; raise an exception if the file contains dependency information for more than one target. This can provide a small sanity check @@ -2453,7 +2445,6 @@ file. -The filename and all of the files listed therein will be interpreted relative to @@ -2473,10 +2464,10 @@ function. Parses one or more strings containing -typical command-line flags for GCC tool chains +typical command-line flags for GCC-style tool chains and returns a dictionary with the flag values separated into the appropriate SCons construction variables. -This is intended as a companion to the +Intended as a companion to the &f-link-env-MergeFlags; method, but allows for the values in the returned dictionary to be modified, if necessary, @@ -2491,11 +2482,20 @@ directly unless you want to manipulate the values.) If the first character in any string is -an exclamation mark (!), +an exclamation mark (!), the rest of the string is executed as a command, and the output from the command is parsed as GCC tool chain command-line flags and added to the resulting dictionary. +This can be used to call a *-config +command typical of the POSIX programming environment +(for example, +pkg-config). +Note that such a comamnd is executed using the +SCons execution environment; +if the command needs additional information, +that information needs to be explcitly provided. +See &f-link-ParseConfig; for more details. diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index 8359405..8bf327a 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -2075,6 +2075,10 @@ def generate(env): orig_backtick = env.backtick class my_backtick: + """mocked backtick routine so command is not actually issued. + + Just returns the string it was given. + """ def __init__(self, save_command, output): self.save_command = save_command self.output = output @@ -2138,6 +2142,32 @@ def generate(env): finally: env.backtick = orig_backtick + # check that we can pass our own function, + # and that it works for both values of unique + + def my_function(myenv, flags, unique=True): + import json + + args = json.loads(flags) + if unique: + myenv.AppendUnique(**args) + else: + myenv.Append(**args) + + json_str = '{"LIBS": ["yyy", "xxx", "yyy"]}' + + env = Environment(LIBS=['xxx']) + env2 = env.Clone() + env.backtick = my_backtick([], json_str) + env2.backtick = my_backtick([], json_str) + + env.ParseConfig("foo", my_function) + assert env['LIBS'] == ['xxx', 'yyy'], env['LIBS'] + + env2.ParseConfig("foo2", my_function, unique=False) + assert env2['LIBS'] == ['xxx', 'yyy', 'xxx', 'yyy'], env2['LIBS'] + + def test_ParseDepends(self): """Test the ParseDepends() method""" test = TestCmd.TestCmd(workdir = '') diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 6e01d2a..4c3f60c 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -223,6 +223,7 @@ def __xml_scan(node, env, path, arg): # Try to call xsltproc xsltproc = env.subst("$DOCBOOK_XSLTPROC") if xsltproc and xsltproc.endswith('xsltproc'): + # TODO: switch to _subproc or subprocess.run call result = env.backtick(' '.join([xsltproc, xsl_file, str(node)])) depfiles = [x.strip() for x in str(result).splitlines() if x.strip() != "" and not x.startswith(" %scons; - + %builders-mod; @@ -21,7 +21,9 @@ is an internal &SCons; object which automatically converts - the options we specified as a string into a list. + the options you specify as a string into a list. @@ -109,7 +111,7 @@ env = Environment() env.Append(CPPPATH=['/include', '/usr/local/include', '/usr/include']) flags = {'CPPPATH': ['/usr/opt/include', '/usr/local/include']} env.MergeFlags(flags) -print(env['CPPPATH']) +print("CPPPATH:", env['CPPPATH']) @@ -124,9 +126,9 @@ print(env['CPPPATH']) [TODO: for when we make CLVar public] is a Python list, not a CLVar, --> - is a normal Python list, - so we must specify its values as a list - in the dictionary we pass to the &MergeFlags; function. + is a normal &Python; list, + so you should give its values as a list + in the dictionary you pass to the &MergeFlags; function. @@ -143,8 +145,8 @@ env = Environment() env.Append(CCFLAGS='-option -O3 -O1') env.Append(CPPPATH=['/include', '/usr/local/include', '/usr/include']) env.MergeFlags('-whatever -I/usr/opt/include -O3 -I/usr/local/include') -print(env['CCFLAGS']) -print(env['CPPPATH']) +print("CCFLAGS:", env['CCFLAGS']) +print("CPPPATH:", env['CPPPATH']) @@ -157,8 +159,8 @@ print(env['CPPPATH']) In the combined example above, &ParseFlags; has sorted the options into their corresponding variables and returned a dictionary for &MergeFlags; to apply - to the construction variables - in the specified construction environment. + to the &consvars; + in the specified &consenv;. diff --git a/doc/user/parseconfig.xml b/doc/user/parseconfig.xml index a07201a..fc9a889 100644 --- a/doc/user/parseconfig.xml +++ b/doc/user/parseconfig.xml @@ -2,7 +2,7 @@ %scons; - + %builders-mod; @@ -21,7 +21,9 @@