From c7fedae70b112c03f8898e4b3091770a07b46f2c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 1 Apr 2019 07:51:43 -0600 Subject: Run all CI builds with -j 2 With the output ordering cleaned up in runtest.py, run all the jobs as two-process to try to lot some more off the build itme. This is a CI-config only change, no code. Signed-off-by: Mats Wichmann --- .appveyor.yml | 2 +- .travis.yml | 47 +++++++++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index cbf7233..d2c63cb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -217,7 +217,7 @@ build_script: # Windows run the tests # NOTE: running powershell from cmd on purpose because it formats the output # correctly - - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}" + - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}" # linux run the tests # unset JAVA_TOOL_OPTIONS because newer java prints this to stderr diff --git a/.travis.yml b/.travis.yml index a8cb940..b0d64e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,13 +27,38 @@ stages: - Coverage - Test +# Note: Travis does not provide a way to specify the order of +# jobs within a Stage, which are "run in parallel", but with +# limitations: from observation four or five are kicked +# off, then additional jobs as initial ones complete. +# We want the slowest jobs in the first batch since the +# faster ones are less than half the time of the slowest, +# we should be able to finish the Test task in the time of the +# slowest job rather than (a fast job + the slowest job). +# Putting the pypy jobs first may help with this, though it's +# apparently not guaranteed. + jobs: include: - &test_job stage: Test - script: python runtest.py -a -t || if [[ $? == 2 ]]; then true; else false; fi + script: python runtest.py -a -t -j 2 || if [[ $? == 2 ]]; then true; else false; fi before_script: skip after_success: skip + python: pypy3 + env: + - PYVER=pypy3 + - PYTHON=pypy3 + sudo: required + + - <<: *test_job + python: pypy + env: + - PYVER=pypy + - PYTHON=pypy + sudo: required + + - <<: *test_job python: 2.7 env: - PYVER=27 @@ -62,24 +87,6 @@ jobs: sudo: required dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) - - &slow_test_job - stage: Test - script: python runtest.py -a -j 4 -t || if [[ $? == 2 ]]; then true; else false; fi - before_script: skip - after_success: skip - python: pypy - env: - - PYVER=pypy - - PYTHON=pypy - sudo: required - - - <<: *slow_test_job - python: pypy3 - env: - - PYVER=pypy3 - - PYTHON=pypy3 - sudo: required - - &coverage_jobs stage: Coverage @@ -128,7 +135,7 @@ jobs: - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi - if (( ${start} == 0 )); then start=1; fi - sed -n ${start},${end}p all_tests > build_tests - - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi + - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests -j 2 || if [[ $? == 2 ]]; then true; else false; fi after_script: - coverage combine -- cgit v0.12 From 11355c20e8779a644f6eb51a9cbf80557c3e905c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 1 Apr 2019 17:40:42 -0600 Subject: [PR #3339] Updates for that failed Since the build with everything-CI-j2 triggered a couple of test fails, include the patches for open-file warnings in those test areas in the hopes it will help. Signed-off-by: Mats Wichmann --- test/Interactive/configure.py | 7 +++---- test/Interactive/option-j.py | 16 ++++++++-------- test/Parallel/duplicate-children.py | 8 ++++---- test/Parallel/duplicate-target.py | 9 ++++++--- test/Parallel/failed-build.py | 3 ++- test/Parallel/multiple-parents.py | 4 ++-- test/Parallel/ref_count.py | 10 +++++----- 7 files changed, 30 insertions(+), 27 deletions(-) diff --git a/test/Interactive/configure.py b/test/Interactive/configure.py index 906a7e9..a7f0735 100644 --- a/test/Interactive/configure.py +++ b/test/Interactive/configure.py @@ -38,10 +38,9 @@ test = TestSCons.TestSCons() test.write('mycc.py', r""" import sys -outfile = open(sys.argv[1], 'wb') -infile = open(sys.argv[2], 'rb') -for l in [l for l in infile.readlines() if l[:7] != '/*c++*/']: - outfile.write(l) +with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: + for l in [l for l in ifp.readlines() if l[:7] != '/*c++*/']: + ofp.write(l) sys.exit(0) """) diff --git a/test/Interactive/option-j.py b/test/Interactive/option-j.py index 279fc51..356a067 100644 --- a/test/Interactive/option-j.py +++ b/test/Interactive/option-j.py @@ -39,10 +39,10 @@ from SCons.Script import * def cat(target, source, env): t = str(target[0]) os.mkdir(t + '.started') - fp = open(t, 'wb') - for s in source: - fp.write(open(str(s), 'rb').read()) - fp.close() + with open(t, 'wb') as ofp: + for s in source: + with open(str(s), 'rb') as ifp: + ofp.write(ifp.read()) os.mkdir(t + '.finished') def must_be_finished(target, source, env, dir): @@ -63,10 +63,10 @@ def must_wait_for_f2_b_out(target, source, env): f2_b_started = 'f2-b.out.started' while not os.path.exists(f2_b_started): time.sleep(1) - fp = open(t, 'wb') - for s in source: - fp.write(open(str(s), 'rb').read()) - fp.close() + with open(t, 'wb') as ofp: + for s in source: + with open(str(s), 'rb') as ifp: + ofp.write(ifp.read()) os.mkdir(t + '.finished') def _f2_a_out_must_not_be_finished(target, source, env): diff --git a/test/Parallel/duplicate-children.py b/test/Parallel/duplicate-children.py index 8b8f4cd..fab1e58 100644 --- a/test/Parallel/duplicate-children.py +++ b/test/Parallel/duplicate-children.py @@ -37,10 +37,10 @@ test = TestSCons.TestSCons() test.write('cat.py', """\ import sys -fp = open(sys.argv[1], 'wb') -for fname in sys.argv[2:]: - fp.write(open(fname, 'rb').read()) -fp.close() +with open(sys.argv[1], 'wb') as ofp: + for fname in sys.argv[2:]: + with open(fname, 'rb') as ifp: + ofp.write(ifp.read()) """) test.write('sleep.py', """\ diff --git a/test/Parallel/duplicate-target.py b/test/Parallel/duplicate-target.py index efe20d9..e947edc 100644 --- a/test/Parallel/duplicate-target.py +++ b/test/Parallel/duplicate-target.py @@ -49,14 +49,15 @@ test.write(['work', 'mycopy.py'], """\ import sys import time time.sleep(int(sys.argv[1])) -open(sys.argv[2], 'wb').write(open(sys.argv[3], 'rb').read()) +with open(sys.argv[2], 'wb') as ofp, open(sys.argv[3], 'rb') as ifp: + ofp.write(ifp.read()) """) test.write(['work', 'mytar.py'], """\ import sys import os.path -fp = open(sys.argv[1], 'wb') +ofp = open(sys.argv[1], 'wb') def visit(dirname): names = os.listdir(dirname) @@ -66,10 +67,12 @@ def visit(dirname): if os.path.isdir(p): visit(p) elif os.path.isfile(p): - fp.write(open(p, 'rb').read()) + with open(p, 'rb') as ifp: + ofp.write(ifp.read()) for s in sys.argv[2:]: visit(s) +ofp.close() """) test.write(['work', 'SConstruct'], """\ diff --git a/test/Parallel/failed-build.py b/test/Parallel/failed-build.py index 2360679..bfdc01d 100644 --- a/test/Parallel/failed-build.py +++ b/test/Parallel/failed-build.py @@ -78,7 +78,8 @@ import os import sys import time os.mkdir('mycopy.started') -open(sys.argv[1], 'wb').write(open(sys.argv[2], 'rb').read()) +with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp: + ofp.write(ifp.read()) for i in [1, 2, 3, 4, 5]: time.sleep(2) if os.path.exists('myfail.exiting'): diff --git a/test/Parallel/multiple-parents.py b/test/Parallel/multiple-parents.py index 7fbcb00..5a52f28 100644 --- a/test/Parallel/multiple-parents.py +++ b/test/Parallel/multiple-parents.py @@ -147,8 +147,8 @@ Default(all) re_error = """\ (scons: \\*\\*\\* \\[failed\\d+] Error 2\\n)|\ (scons: \\*\\*\\* \\[missing\\d+] Source `MissingSrc' not found, needed by target `missing\\d+'\\.( Stop\\.)?\\n)|\ -(scons: \\*\\*\\* \\[\\w+] Build interrupted\.\\n)|\ -(scons: Build interrupted\.\\n)\ +(scons: \\*\\*\\* \\[\\w+] Build interrupted\\.\\n)|\ +(scons: Build interrupted\\.\\n)\ """ re_errors = "(" + re_error + ")+" diff --git a/test/Parallel/ref_count.py b/test/Parallel/ref_count.py index 7ce5910..8a31bf3 100644 --- a/test/Parallel/ref_count.py +++ b/test/Parallel/ref_count.py @@ -74,12 +74,12 @@ while args: time.sleep(int(args.pop(0))) contents = '' for ifile in args: - contents = contents + open(ifile, 'r').read() + with open(ifile, 'r') as ifp: + contents = contents + ifp.read() for ofile in outputs: - ofp = open(ofile, 'w') - ofp.write('%s: building from %s\\n' % (ofile, " ".join(args))) - ofp.write(contents) - ofp.close() + with open(ofile, 'w') as ofp: + ofp.write('%s: building from %s\\n' % (ofile, " ".join(args))) + ofp.write(contents) """) # -- cgit v0.12 From 75f738bda9cae575342cab2d843f71aea0007aef Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 29 Dec 2018 12:22:44 -0700 Subject: Give sconsign a default filename. sconsign required filename(s) or directory name(s) to do anything, in their absence it just quits silently. Change so if filename argument omitted, use the same default as scons - .sconsign.dblite. print something in case of bad options. add an extra info line in case the sconsign cannot be read due to pickle protocol (when py2 used in a place where scons previously run with py3) Tweak the manpage a bit. Signed-off-by: Mats Wichmann --- doc/man/sconsign.xml | 56 +++++++++++++++++++------------ src/CHANGES.txt | 2 ++ src/script/sconsign.py | 89 ++++++++++++++++++++++++++++---------------------- 3 files changed, 88 insertions(+), 59 deletions(-) diff --git a/doc/man/sconsign.xml b/doc/man/sconsign.xml index aad2cd9..03b6351 100644 --- a/doc/man/sconsign.xml +++ b/doc/man/sconsign.xml @@ -37,15 +37,14 @@ sconsign -print SCons .sconsign file information +print SCons signature file information sconsign options - file - ... + file @@ -54,30 +53,38 @@ The sconsign command -displays the contents of one or more -.sconsign +displays the contents of one or more signature +("sconsign") files specified by the user. By default, sconsign dumps the entire contents of the specified file(s). -Each entry is printed in the following format: +Without the verbose option, +each entry is printed in the following format: - file: signature timestamp length - implicit_dependency_1: signature timestamp length - implicit_dependency_2: signature timestamp length - action_signature [action string] + +file: signature timestamp length + implicit_dependency_1: signature timestamp length + implicit_dependency_2: signature timestamp length + ... + action_signature [action string] + None is printed -in place of any missing timestamp, bsig, or csig +in place of any missing timestamp, build signature ("bsig"), +or content signature ("csig") values for any entry or any of its dependencies. If the entry has no implicit dependencies, or no build action, -the lines are simply omitted. +the lines are simply omitted. +The verbose option expands the display into a more human +readable format. + By default, sconsign @@ -90,7 +97,7 @@ signature entries for more than one directory (that is, was specified by the -SConsignFile () +SConsignFile function). Any file @@ -103,13 +110,21 @@ for a single directory. If neither of those is true, sconsign attempts to guess the format. -If that does not work, +If that does not work, an explicit format may be specified using the or -options. +options. + + +If there are no +file +arguments, the name +.sconsign.dblite +is assumed. + @@ -175,7 +190,7 @@ Legal values are dbm (the DBM format used when the -SConsignFile() +SConsignFile function is used) or sconsign @@ -197,7 +212,7 @@ file in each directory). -i, --implicit Prints the list of cached implicit dependencies -for all entries or the the specified entries. +for all entries or for the specified entries. @@ -206,8 +221,8 @@ for all entries or the the specified entries. Prints a pretty-printed representation of the raw Python dictionary that holds -build information about individual entry -(both the entry itself or its implicit dependencies). +build information about individual entries +(both the entry itself and its implicit dependencies). An entry's build action is still printed in its usual format. @@ -244,7 +259,8 @@ for all entries or the specified entries. SCONS_LIB_DIR Specifies the directory that contains the SCons Python module directory -(e.g. /home/aroach/scons-src-0.01/src/engine). +(e.g. +/home/aroach/scons-src-0.01/src/engine). on the command line. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 15e8b5f..84a8056 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -11,6 +11,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - scons-time takes more care closing files and uses safer mkdtemp to avoid possible races on multi-job runs. - Use importlib to dynamically load tool and platform modules instead of imp module + - sconsign: default to .sconsign.dblite if no filename is specified. + Be more informative in case of unsupported pickle protocol (py2 only). From John Doe: diff --git a/src/script/sconsign.py b/src/script/sconsign.py index cb2b5c6..6ba0a7c 100644 --- a/src/script/sconsign.py +++ b/src/script/sconsign.py @@ -185,6 +185,7 @@ import SCons.compat try: import whichdb + whichdb = whichdb.whichdb except ImportError as e: from dbm import whichdb @@ -199,9 +200,8 @@ def my_whichdb(filename): if filename[-7:] == ".dblite": return "SCons.dblite" try: - f = open(filename + ".dblite", "rb") - f.close() - return "SCons.dblite" + with open(filename + ".dblite", "rb"): + return "SCons.dblite" except IOError: pass return _orig_whichdb(filename) @@ -248,9 +248,8 @@ Readable = 0 Warns = 0 - def default_mapper(entry, name): - ''' + """ Stringify an entry that doesn't have an explicit mapping. Args: @@ -259,7 +258,7 @@ def default_mapper(entry, name): Returns: str - ''' + """ try: val = eval("entry." + name) except: @@ -273,7 +272,7 @@ def default_mapper(entry, name): def map_action(entry, _): - ''' + """ Stringify an action entry and signature. Args: @@ -282,7 +281,7 @@ def map_action(entry, _): Returns: str - ''' + """ try: bact = entry.bact bactsig = entry.bactsig @@ -290,8 +289,9 @@ def map_action(entry, _): return None return '%s [%s]' % (bactsig, bact) + def map_timestamp(entry, _): - ''' + """ Stringify a timestamp entry. Args: @@ -300,7 +300,7 @@ def map_timestamp(entry, _): Returns: str - ''' + """ try: timestamp = entry.timestamp except AttributeError: @@ -310,8 +310,9 @@ def map_timestamp(entry, _): else: return str(timestamp) + def map_bkids(entry, _): - ''' + """ Stringify an implicit entry. Args: @@ -320,7 +321,7 @@ def map_bkids(entry, _): Returns: str - ''' + """ try: bkids = entry.bsources + entry.bdepends + entry.bimplicit bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs @@ -449,8 +450,8 @@ class Do_SConsignDB(object): def __call__(self, fname): # The *dbm modules stick their own file suffixes on the names - # that are passed in. This is causes us to jump through some - # hoops here to be able to allow the user + # that are passed in. This causes us to jump through some + # hoops here. try: # Try opening the specified file name. Example: # SPECIFIED OPENED BY self.dbm.open() @@ -462,16 +463,17 @@ class Do_SConsignDB(object): print_e = e try: # That didn't work, so try opening the base name, - # so that if the actually passed in 'sconsign.dblite' + # so that if they actually passed in 'sconsign.dblite' # (for example), the dbm module will put the suffix back # on for us and open it anyway. db = self.dbm.open(os.path.splitext(fname)[0], "r") except (IOError, OSError): # That didn't work either. See if the file name - # they specified just exists (independent of the dbm + # they specified even exists (independent of the dbm # suffix-mangling). try: - open(fname, "r") + with open(fname, "rb"): + pass # this is a touch only, we don't use it here. except (IOError, OSError) as e: # Nope, that file doesn't even exist, so report that # fact back. @@ -487,6 +489,9 @@ class Do_SConsignDB(object): except Exception as e: sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" % (self.dbm_name, fname, e)) + exc_type, _, _ = sys.exc_info() + if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): + sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") return if Print_Directories: @@ -513,23 +518,23 @@ class Do_SConsignDB(object): def Do_SConsignDir(name): try: - fp = open(name, 'rb') + with open(name, 'rb') as fp: + try: + sconsign = SCons.SConsign.Dir(fp) + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + err = "sconsign: ignoring invalid .sconsign file `%s'\n" % (name) + sys.stderr.write(err) + return + except Exception as e: + err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) + sys.stderr.write(err) + return + printentries(sconsign.entries, args[0]) except (IOError, OSError) as e: sys.stderr.write("sconsign: %s\n" % e) return - try: - sconsign = SCons.SConsign.Dir(fp) - except KeyboardInterrupt: - raise - except pickle.UnpicklingError: - err = "sconsign: ignoring invalid .sconsign file `%s'\n" % (name) - sys.stderr.write(err) - return - except Exception as e: - err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) - sys.stderr.write(err) - return - printentries(sconsign.entries, args[0]) ############################################################################## @@ -537,7 +542,7 @@ def Do_SConsignDir(name): import getopt helpstr = """\ -Usage: sconsign [OPTIONS] FILE [...] +Usage: sconsign [OPTIONS] [FILE ...] Options: -a, --act, --action Print build action information. -c, --csig Print content signature information. @@ -553,13 +558,17 @@ Options: -v, --verbose Verbose, describe each field. """ -opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", - ['act', 'action', - 'csig', 'dir=', 'entry=', - 'format=', 'help', 'implicit', - 'raw', 'readable', - 'size', 'timestamp', 'verbose']) - +try: + opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", + ['act', 'action', + 'csig', 'dir=', 'entry=', + 'format=', 'help', 'implicit', + 'raw', 'readable', + 'size', 'timestamp', 'verbose']) +except getopt.GetoptError as err: + sys.stderr.write(str(err) + '\n') + print(helpstr) + sys.exit(2) for o, a in opts: if o in ('-a', '--act', '--action'): @@ -614,6 +623,8 @@ if Do_Call: for a in args: Do_Call(a) else: + if not args: + args = [".sconsign.dblite"] for a in args: dbm_name = whichdb(a) if dbm_name: -- cgit v0.12 From d1d49cf9061ebee5255868b54ae0e0d3795dc487 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 29 Sep 2018 09:32:06 -0600 Subject: Tweaks to packaging tests: avoid default tool list Signed-off-by: Mats Wichmann --- test/packaging/multiple-subdirs.py | 2 +- test/packaging/option--package-type.py | 2 +- test/packaging/place-files-in-subdirectory.py | 6 +++--- test/packaging/strip-install-dir.py | 2 +- test/packaging/tar/bz2_packaging.py | 2 +- test/packaging/tar/gz.py | 2 +- test/packaging/tar/xz_packaging.py | 2 +- test/packaging/use-builddir.py | 4 ++-- test/packaging/zip.py | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/packaging/multiple-subdirs.py b/test/packaging/multiple-subdirs.py index 66b81a3..27b81a2 100644 --- a/test/packaging/multiple-subdirs.py +++ b/test/packaging/multiple-subdirs.py @@ -42,7 +42,7 @@ if not tar: test.subdir('one', 'two', 'three') test.write('SConstruct', """\ -env = Environment(tools=['default', 'packaging']) +env = Environment(tools=['packaging', 'filesystem', 'tar']) Export('env') SConscript(dirs = ['one', 'two', 'three']) """) diff --git a/test/packaging/option--package-type.py b/test/packaging/option--package-type.py index 7ff8535..b9ed05e 100644 --- a/test/packaging/option--package-type.py +++ b/test/packaging/option--package-type.py @@ -53,7 +53,7 @@ test.write( 'main', '' ) test.write('SConstruct', """ # -*- coding: iso-8859-15 -*- -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar', 'rpm']) env.Prepend(RPM = 'TAR_OPTIONS=--wildcards ') env.Append(RPMFLAGS = r' --buildroot %(rpm_build_root)s') prog=env.Install( '/bin', 'main' ) diff --git a/test/packaging/place-files-in-subdirectory.py b/test/packaging/place-files-in-subdirectory.py index f7224ba..0eb791f 100644 --- a/test/packaging/place-files-in-subdirectory.py +++ b/test/packaging/place-files-in-subdirectory.py @@ -48,7 +48,7 @@ test.subdir('src') test.write('src/main.c', '') test.write('SConstruct', """ -env = Environment(tools=['default', 'packaging']) +env = Environment(tools=['packaging', 'filesystem', 'zip']) env.Package( NAME = 'libfoo', PACKAGEROOT = 'libfoo', PACKAGETYPE = 'src_zip', @@ -70,7 +70,7 @@ test.subdir('src') test.write('src/main.c', '') test.write('SConstruct', """ -env = Environment(tools=['default', 'packaging']) +env = Environment(tools=['packaging', 'filesystem', 'zip']) env.Package( NAME = 'libfoo', VERSION = '1.2.3', PACKAGETYPE = 'src_zip', @@ -93,7 +93,7 @@ test.subdir('temp') test.write('src/main.c', '') test.write('SConstruct', """ -env = Environment(tools=['default', 'packaging']) +env = Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( NAME = 'libfoo', VERSION = '1.2.3', PACKAGETYPE = 'src_targz', diff --git a/test/packaging/strip-install-dir.py b/test/packaging/strip-install-dir.py index b81a6b4..7f41c20 100644 --- a/test/packaging/strip-install-dir.py +++ b/test/packaging/strip-install-dir.py @@ -43,7 +43,7 @@ if not tar: test.write( 'main.c', '' ) test.write('SConstruct', """ prog = Install( '/bin', 'main.c' ) -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( NAME = 'foo', VERSION = '1.2.3', source = [ prog ], diff --git a/test/packaging/tar/bz2_packaging.py b/test/packaging/tar/bz2_packaging.py index 2a8b506..875f1c9 100644 --- a/test/packaging/tar/bz2_packaging.py +++ b/test/packaging/tar/bz2_packaging.py @@ -54,7 +54,7 @@ int main( int argc, char* argv[] ) test.write('SConstruct', """ Program( 'src/main.c' ) -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( PACKAGETYPE = 'src_tarbz2', target = 'src.tar.bz2', PACKAGEROOT = 'test', diff --git a/test/packaging/tar/gz.py b/test/packaging/tar/gz.py index 05661b7..07d5b52 100644 --- a/test/packaging/tar/gz.py +++ b/test/packaging/tar/gz.py @@ -50,7 +50,7 @@ int main( int argc, char* argv[] ) test.write('SConstruct', """ Program( 'src/main.c' ) -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( PACKAGETYPE = 'src_targz', target = 'src.tar.gz', PACKAGEROOT = 'test', diff --git a/test/packaging/tar/xz_packaging.py b/test/packaging/tar/xz_packaging.py index 194b110..f12292e 100644 --- a/test/packaging/tar/xz_packaging.py +++ b/test/packaging/tar/xz_packaging.py @@ -54,7 +54,7 @@ int main( int argc, char* argv[] ) test.write('SConstruct', """ Program( 'src/main.c' ) -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( PACKAGETYPE = 'src_tarxz', target = 'src.tar.xz', PACKAGEROOT = 'test', diff --git a/test/packaging/use-builddir.py b/test/packaging/use-builddir.py index 9ad7aa4..812f2d6 100644 --- a/test/packaging/use-builddir.py +++ b/test/packaging/use-builddir.py @@ -50,7 +50,7 @@ test.write('src/main.c', '') test.write('SConstruct', """ VariantDir('build', 'src') -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'zip']) env.Package( NAME = 'libfoo', PACKAGEROOT = 'build/libfoo', VERSION = '1.2.3', @@ -74,7 +74,7 @@ test.write('src/main.c', '') test.write('SConstruct', """ VariantDir('build', 'src') -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'tar']) env.Package( NAME = 'libfoo', VERSION = '1.2.3', PAKCAGETYPE = 'src_targz', diff --git a/test/packaging/zip.py b/test/packaging/zip.py index 89f3074..527ad00 100644 --- a/test/packaging/zip.py +++ b/test/packaging/zip.py @@ -51,7 +51,7 @@ int main( int argc, char* argv[] ) test.write('SConstruct', """ Program( 'src/main.c' ) -env=Environment(tools=['default', 'packaging']) +env=Environment(tools=['packaging', 'filesystem', 'zip']) env.Package( PACKAGETYPE = 'src_zip', target = 'src.zip', PACKAGEROOT = 'test', -- cgit v0.12 From fd4a8f7c9383095a0e508bac937b6122dd5317d6 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 28 Mar 2019 05:17:52 -0600 Subject: [#3336] do not add not-found tool paths When tool modules initialize, they check paths to decide if the underlying tool is actually present. The current checker adds any "default paths" the caller may have supplied (locations like mingw, cygwin, chocolatey install locations, etc.); if there is match from this list, any previous default paths are also kept. To avoid keeping these non-matching paths, restore the original PATH; the caller is responsible for adding to PATH if necessary. Docstring now says so. Note lex and yacc tool modules seem to expect the path-modifying behavior that's being gotten rid of - so they preseve the path first and restore it after. The change here won't break those, but makes the extra behavior unneeded - could remove it. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 2 ++ src/engine/SCons/Tool/__init__.py | 25 ++++++++++++++----------- src/engine/SCons/Tool/swig.py | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 84a8056..d07b4ad 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -13,6 +13,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Use importlib to dynamically load tool and platform modules instead of imp module - sconsign: default to .sconsign.dblite if no filename is specified. Be more informative in case of unsupported pickle protocol (py2 only). + - Fix issue #3336 - on Windows, paths were being added to PATH even if + tools were not found in those paths. From John Doe: diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 8fbd587..152e186 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -1318,28 +1318,31 @@ def tool_list(platform, env): def find_program_path(env, key_program, default_paths=[]): """ - Find the location of key_program and then return the path it was located at. - Checking the default install locations. - Mainly for windows where tools aren't all installed in /usr/bin,etc - :param env: Current Environment() - :param key_program: Program we're using to locate the directory to add to PATH. + Find the location of a tool using various means. + + Mainly for windows where tools aren't all installed in /usr/bin, etc. + + :param env: Current Construction Environment. + :param key_program: Tool to locate. + :param default_paths: List of additional paths this tool might be found in. """ # First search in the SCons path path = env.WhereIs(key_program) - if (path): + if path: return path - # then the OS path: + + # Then in the OS path path = SCons.Util.WhereIs(key_program) - if (path): + if path: return path - # If that doesn't work try default location for mingw + # Finally, add the defaults and check again. Do not change + # ['ENV']['PATH'] permananetly, the caller can do that if needed. save_path = env['ENV']['PATH'] for p in default_paths: env.AppendENVPath('PATH', p) path = env.WhereIs(key_program) - if not path: - env['ENV']['PATH'] = save_path + env['ENV']['PATH'] = save_path return path # Local Variables: diff --git a/src/engine/SCons/Tool/swig.py b/src/engine/SCons/Tool/swig.py index c6abefb..6ed9d82 100644 --- a/src/engine/SCons/Tool/swig.py +++ b/src/engine/SCons/Tool/swig.py @@ -184,9 +184,10 @@ def generate(env): from SCons.Platform.mingw import MINGW_DEFAULT_PATHS from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS + from SCons.Platform.win32 import CHOCO_DEFAULT_PATH if sys.platform == 'win32': - swig = SCons.Tool.find_program_path(env, 'swig', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS + [r'C:\ProgramData\chocolatey\bin'] ) + swig = SCons.Tool.find_program_path(env, 'swig', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS + CHOCO_DEFAULT_PATH) if swig: swig_bin_dir = os.path.dirname(swig) env.AppendENVPath('PATH', swig_bin_dir) -- cgit v0.12 From 77f302bcc7cff7faca0944ac6848de1c82c487d9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 13 Apr 2019 11:38:28 -0600 Subject: [PR #3337] clean up lex and yacc tools Remove now unneeded code to save/restore the path, since the routine now does not modify the path. Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/lex.py | 19 +++++++++++-------- src/engine/SCons/Tool/yacc.py | 27 +++++++++++++++++---------- test/LEX/live_mingw.py | 2 +- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index a63ddc9..91d50fd 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -70,11 +70,15 @@ def lexEmitter(target, source, env): def get_lex_path(env, append_paths=False): """ - Find the a path containing the lex or flex binaries. If a construction - environment is passed in then append the path to the ENV PATH. + Find the path to the lex tool, searching several possible names + + Only called in the Windows case, so the default_path + can be Windows-specific + + :param env: current construction environment + :param append_paths: if set, add the path to the tool to PATH + :return: path to lex tool, if found """ - # save existing path to reset if we don't want to append any paths - envPath = env['ENV']['PATH'] bins = ['flex', 'lex', 'win_flex'] for prog in bins: @@ -83,9 +87,7 @@ def get_lex_path(env, append_paths=False): prog, default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) if bin_path: - if not append_paths: - env['ENV']['PATH'] = envPath - else: + if append_paths: env.AppendENVPath('PATH', os.path.dirname(bin_path)) return bin_path SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') @@ -113,7 +115,8 @@ def generate(env): env["LEXFLAGS"] = SCons.Util.CLVar("") if sys.platform == 'win32': - get_lex_path(env, append_paths=True) + # ignore the return - we do not need the full path here + _ = get_lex_path(env, append_paths=True) env["LEX"] = env.Detect(['flex', 'lex', 'win_flex']) if not env.get("LEXUNISTD"): env["LEXUNISTD"] = SCons.Util.CLVar("") diff --git a/src/engine/SCons/Tool/yacc.py b/src/engine/SCons/Tool/yacc.py index 5db49d3..07ba8da 100644 --- a/src/engine/SCons/Tool/yacc.py +++ b/src/engine/SCons/Tool/yacc.py @@ -100,11 +100,15 @@ def yyEmitter(target, source, env): def get_yacc_path(env, append_paths=False): """ - Find the a path containing the lex or flex binaries. If a construction - environment is passed in then append the path to the ENV PATH. + Find the path to the yacc tool, searching several possible names + + Only called in the Windows case, so the default_path + can be Windows-specific + + :param env: current construction environment + :param append_paths: if set, add the path to the tool to PATH + :return: path to yacc tool, if found """ - # save existing path to reset if we don't want to append any paths - envPath = env['ENV']['PATH'] bins = ['bison', 'yacc', 'win_bison'] for prog in bins: @@ -113,12 +117,11 @@ def get_yacc_path(env, append_paths=False): prog, default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) if bin_path: - if not append_paths: - env['ENV']['PATH'] = envPath - else: + if append_paths: env.AppendENVPath('PATH', os.path.dirname(bin_path)) return bin_path - SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH') + SCons.Warnings.Warning('yacc tool requested, but yacc or bison binary not found in ENV PATH') + def generate(env): """Add Builders and construction variables for yacc to an Environment.""" @@ -148,7 +151,8 @@ def generate(env): SCons.Warnings.Warning('yacc tool requested, but bison binary not found in ENV PATH') if sys.platform == 'win32': - get_yacc_path(env, append_paths=True) + # ignore the return - we do not need the full path here + _ = get_yacc_path(env, append_paths=True) env["YACC"] = env.Detect(['bison', 'yacc', 'win_bison']) else: env["YACC"] = env.Detect(["bison", "yacc"]) @@ -162,7 +166,10 @@ def generate(env): env['YACCVCGFILESUFFIX'] = '.vcg' def exists(env): - return env.Detect(['bison', 'yacc']) + if sys.platform == 'win32': + return get_yacc_path(env) + else: + return env.Detect(['bison', 'yacc']) # Local Variables: # tab-width:4 diff --git a/test/LEX/live_mingw.py b/test/LEX/live_mingw.py index 13e2342..d535065 100644 --- a/test/LEX/live_mingw.py +++ b/test/LEX/live_mingw.py @@ -41,7 +41,7 @@ if sys.platform != 'win32': test.skip_test('Not windows environment; skipping test.\n') if not test.where_is('gcc'): - test.skip_test('No mingw or cygwin on windows; skipping test.\n') + test.skip_test('No mingw or cygwin build environment found; skipping test.\n') lex = test.where_is('lex') or test.where_is('flex') -- cgit v0.12 From a2a1fede9770daee1ff6d14d8870000640343fde Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 28 Mar 2019 05:19:18 -0600 Subject: Fix problems with jdk detection The java tool common routine finds a jdk by doing a filesystem glob. This had a problem on windows in the case a specific version is requested, because the format of name of the jdk directory has changed with JDK 9 - there is a dash between jdk and the version string. The glob which does not attempt to match a version was general enough not to trip on this, but with a version to match it would never match jdk-9 or higher. The test harness then asks the found javac what version it is, and the parsing of that did not work as expected once version numbers became double-digit, as the regex was for a single digit followed by a dot. The outcome is for 11.0.2 we get back '11' instead of '11.0'. Change the regex to match any number of digits followed by dot. The Repository/RMIC.py change is to align with an earlier change to Java/RMIC.py, but the may not be needed after the change to regex just described. Clean up some of the Java tool routines for consistency (there was no functional change outside of JavaCommon.py) Docstrings added or updated in several places. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 3 ++ src/engine/SCons/Tool/JavaCommon.py | 71 ++++++++++++++++++++++--------------- src/engine/SCons/Tool/jar.py | 6 ++-- src/engine/SCons/Tool/javac.py | 8 ++--- src/engine/SCons/Tool/javah.py | 6 ++-- src/engine/SCons/Tool/rmic.py | 6 ++-- test/Repository/RMIC.py | 6 ++++ testing/framework/TestSCons.py | 62 +++++++++++++++++++++++++++----- 8 files changed, 117 insertions(+), 51 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 84a8056..bb08e4c 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -13,6 +13,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Use importlib to dynamically load tool and platform modules instead of imp module - sconsign: default to .sconsign.dblite if no filename is specified. Be more informative in case of unsupported pickle protocol (py2 only). + - more fixes for newer Java versions (since 9): handle new jdk directory + naming (jdk-X.Y instead of jdkX.Y) on Windows; handle two-digit major + version. Docstrings improved. From John Doe: diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py index 853f7f2..c77ace6 100644 --- a/src/engine/SCons/Tool/JavaCommon.py +++ b/src/engine/SCons/Tool/JavaCommon.py @@ -70,7 +70,7 @@ if java_parsing: def __init__(self, version=default_java_version): if not version in ('1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', - '1.8', '5', '6', '9.0', '10.0', '11.0'): + '1.8', '5', '6', '9.0', '10.0', '11.0', '12.0'): msg = "Java version %s not supported" % version raise NotImplementedError(msg) @@ -178,7 +178,7 @@ if java_parsing: if self.version in ('1.1', '1.2', '1.3', '1.4'): clazz = self.listClasses[0] self.listOutputs.append('%s$%d' % (clazz, self.nextAnon)) - elif self.version in ('1.5', '1.6', '1.7', '1.8', '5', '6', '9.0', '10.0', '11.0'): + elif self.version in ('1.5', '1.6', '1.7', '1.8', '5', '6', '9.0', '10.0', '11.0', '12.0'): self.stackAnonClassBrackets.append(self.brackets) className = [] className.extend(self.listClasses) @@ -395,30 +395,27 @@ else: return os.path.split(fn) - -java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk%s*/bin' +# Glob patterns for use in finding where the JDK is. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +# For now only used for Windows. java_win32_dir_glob = 'C:/Program Files*/Java/jdk*/bin' - -java_macos_include_dir = '/System/Library/Frameworks/JavaVM.framework/Headers/' -java_macos_version_include_dir = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' - -java_linux_include_dirs = ['/usr/lib/jvm/default-java/include', - '/usr/lib/jvm/java-*/include'] -# Need to match path like below (from Centos 7) -# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ -java_linux_version_include_dirs = ['/usr/lib/jvm/java-*-sun-%s*/include', - '/usr/lib/jvm/java-%s*-openjdk*/include', - '/usr/java/jdk%s*/include'] - - +# On windows, since Java 9, there is a dash between 'jdk' and the version +# string that wasn't there before. this glob should catch either way. +java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk*%s*/bin' def get_java_install_dirs(platform, version=None): """ - Using patterns above find the java jdk install dir - :param platform: + Find the java jdk installation directories. + + This list is intended to supply as "default paths" for use when looking + up actual java binaries. + + :param platform: selector for search algorithm. :param version: If specified, only look for java sdk's of this version :return: list of default paths for java. """ + paths = [] if platform == 'win32': if version: @@ -426,25 +423,45 @@ def get_java_install_dirs(platform, version=None): else: paths = glob.glob(java_win32_dir_glob) else: - # do nothing for now + # other platforms, do nothing for now pass - paths=sorted(paths) + return sorted(paths) - return paths + +# Glob patterns for use in finding where the JDK headers are. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +java_macos_include_dir = '/System/Library/Frameworks/JavaVM.framework/Headers/' +java_macos_version_include_dir = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' + +java_linux_include_dirs = [ + '/usr/lib/jvm/default-java/include', + '/usr/lib/jvm/java-*/include' +] +# Need to match path like below (from Centos 7) +# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ +java_linux_version_include_dirs = [ + '/usr/lib/jvm/java-*-sun-%s*/include', + '/usr/lib/jvm/java-%s*-openjdk*/include', + '/usr/java/jdk%s*/include' +] def get_java_include_paths(env, javac, version): """ - Return java include paths - :param platform: - :param javac: - :return: + Find java include paths for JNI building. + + :param env: construction environment, used to extract platform. + :param javac: path to detected javac. + :return: list of paths. """ + paths = [] if not javac: # there are no paths if we've not detected javac. pass elif env['PLATFORM'] == 'win32': + # on Windows, we have the right path to javac, so look locally javac_bin_dir = os.path.dirname(javac) java_inc_dir = os.path.normpath(os.path.join(javac_bin_dir, '..', 'include')) paths = [java_inc_dir, os.path.join(java_inc_dir, 'win32')] @@ -469,8 +486,6 @@ def get_java_include_paths(env, javac, version): return paths - - # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Tool/jar.py b/src/engine/SCons/Tool/jar.py index e0a6a69..02bda57 100644 --- a/src/engine/SCons/Tool/jar.py +++ b/src/engine/SCons/Tool/jar.py @@ -209,9 +209,9 @@ def generate(env): env.AddMethod(Jar) if env['PLATFORM'] == 'win32': - # Ensure that we have a proper path for clang - jar = SCons.Tool.find_program_path(env, 'jar', - default_paths=get_java_install_dirs(env['PLATFORM'])) + # Ensure that we have a proper path for jar + paths = get_java_install_dirs('win32') + jar = SCons.Tool.find_program_path(env, 'jar', default_paths=paths) if jar: jar_bin_dir = os.path.dirname(jar) env.AppendENVPath('PATH', jar_bin_dir) diff --git a/src/engine/SCons/Tool/javac.py b/src/engine/SCons/Tool/javac.py index 8d98b54..537913f 100644 --- a/src/engine/SCons/Tool/javac.py +++ b/src/engine/SCons/Tool/javac.py @@ -210,15 +210,15 @@ def generate(env): version = env.get('JAVAVERSION', None) - javac = SCons.Tool.find_program_path(env, 'javac') if env['PLATFORM'] == 'win32': # Ensure that we have a proper path for javac - paths=get_java_install_dirs(env['PLATFORM'], version=version) - javac = SCons.Tool.find_program_path(env, 'javac', - default_paths=paths) + paths = get_java_install_dirs('win32', version=version) + javac = SCons.Tool.find_program_path(env, 'javac', default_paths=paths) if javac: javac_bin_dir = os.path.dirname(javac) env.AppendENVPath('PATH', javac_bin_dir) + else: + javac = SCons.Tool.find_program_path(env, 'javac') env['JAVAINCLUDES'] = get_java_include_paths(env, javac, version) diff --git a/src/engine/SCons/Tool/javah.py b/src/engine/SCons/Tool/javah.py index f514479..80f8a6b 100644 --- a/src/engine/SCons/Tool/javah.py +++ b/src/engine/SCons/Tool/javah.py @@ -123,9 +123,9 @@ def generate(env): java_javah.emitter = emit_java_headers if env['PLATFORM'] == 'win32': - # Ensure that we have a proper path for clang - javah = SCons.Tool.find_program_path(env, 'javah', - default_paths=get_java_install_dirs(env['PLATFORM'])) + # Ensure that we have a proper path for javah + paths = get_java_install_dirs('win32') + javah = SCons.Tool.find_program_path(env, 'javah', default_paths=paths) if javah: javah_bin_dir = os.path.dirname(javah) env.AppendENVPath('PATH', javah_bin_dir) diff --git a/src/engine/SCons/Tool/rmic.py b/src/engine/SCons/Tool/rmic.py index 173ef5f..5c7a040 100644 --- a/src/engine/SCons/Tool/rmic.py +++ b/src/engine/SCons/Tool/rmic.py @@ -110,11 +110,9 @@ def generate(env): if env['PLATFORM'] == 'win32': version = env.get('JAVAVERSION', None) - default_paths=get_java_install_dirs(env['PLATFORM'], version=version) - # Ensure that we have a proper path for rmic - rmic = SCons.Tool.find_program_path(env, 'rmic', default_paths=default_paths) - + paths = get_java_install_dirs('win32', version=version) + rmic = SCons.Tool.find_program_path(env, 'rmic', default_paths=paths) # print("RMIC: %s"%rmic) if rmic: rmic_bin_dir = os.path.dirname(rmic) diff --git a/test/Repository/RMIC.py b/test/Repository/RMIC.py index 433890f..3eeab9b 100644 --- a/test/Repository/RMIC.py +++ b/test/Repository/RMIC.py @@ -46,6 +46,12 @@ if java_version.count('.') == 1: curver = (int(major), int(minor)) except: pass +elif java_version.count('.') == 0: + # java 11? + try: + curver = (int(java_version), 0) + except: + pass # Check the version of the found Java compiler. # If it's 1.8 or higher, we skip the further RMIC test diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 91a443e..23e8753 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -682,6 +682,9 @@ class TestSCons(TestCommon): """ Initialize with a default external environment that uses a local Java SDK in preference to whatever's found in the default PATH. + + :param version: if set, match only that version + :return: the new env. """ if not self.external: try: @@ -698,11 +701,11 @@ class TestSCons(TestCommon): if version: if sys.platform == 'win32': patterns = [ - 'C:/Program Files*/Java/jdk%s*/bin'%version, + 'C:/Program Files*/Java/jdk*%s*/bin' % version, ] else: patterns = [ - '/usr/java/jdk%s*/bin' % version, + '/usr/java/jdk%s*/bin' % version, '/usr/lib/jvm/*-%s*/bin' % version, '/usr/local/j2sdk%s*/bin' % version, ] @@ -727,7 +730,10 @@ class TestSCons(TestCommon): def java_where_includes(self,version=None): """ - Return java include paths compiling java jni code + Find include path needed for compiling java jni code. + + :param version: if set, match only that version + :return: path to java headers """ import sys @@ -761,6 +767,14 @@ class TestSCons(TestCommon): return result def java_where_java_home(self, version=None): + """ + Find path to what would be JAVA_HOME. + + SCons does not read JAVA_HOME from the environment, so deduce it. + + :param version: if set, match only that version + :return: path where JDK components live + """ if sys.platform[:6] == 'darwin': # osx 10.11, 10.12 home_tool = '/usr/libexec/java_home' @@ -807,6 +821,12 @@ class TestSCons(TestCommon): self.skip_test("Could not find Java " + java_bin_name + ", skipping test(s).\n") def java_where_jar(self, version=None): + """ + Find java archiver jar. + + :param version: if set, match only that version + :return: path to jar + """ ENV = self.java_ENV(version) if self.detect_tool('jar', ENV=ENV): where_jar = self.detect('JAR', 'jar', ENV=ENV) @@ -821,7 +841,10 @@ class TestSCons(TestCommon): def java_where_java(self, version=None): """ - Return a path to the java executable. + Find java executable. + + :param version: if set, match only that version + :return: path to the java rutime """ ENV = self.java_ENV(version) where_java = self.where_is('java', ENV['PATH']) @@ -835,7 +858,10 @@ class TestSCons(TestCommon): def java_where_javac(self, version=None): """ - Return a path to the javac compiler. + Find java compiler. + + :param version: if set, match only that version + :return: path to javac """ ENV = self.java_ENV(version) if self.detect_tool('javac'): @@ -851,15 +877,17 @@ class TestSCons(TestCommon): arguments = '-version', stderr=None, status=None) + # Note recent versions output version info to stdout instead of stderr if version: - if self.stderr().find('javac %s' % version) == -1: + verf = 'javac %s' % version + if self.stderr().find(verf) == -1 and self.stdout().find(verf) == -1: fmt = "Could not find javac for Java version %s, skipping test(s).\n" self.skip_test(fmt % version) else: - m = re.search(r'javac (\d\.*\d)', self.stderr()) - # Java 11 outputs this to stdout + version_re = r'javac (\d*\.*\d)' + m = re.search(version_re, self.stderr()) if not m: - m = re.search(r'javac (\d\.*\d)', self.stdout()) + m = re.search(version_re, self.stdout()) if m: version = m.group(1) @@ -873,6 +901,16 @@ class TestSCons(TestCommon): return where_javac, version def java_where_javah(self, version=None): + """ + Find java header generation tool. + + TODO issue #3347 since JDK10, there is no separate javah command, + 'javac -h' is used. We should not return a javah from a different + installed JDK - how to detect and what to return in this case? + + :param version: if set, match only that version + :return: path to javah + """ ENV = self.java_ENV(version) if self.detect_tool('javah'): where_javah = self.detect('JAVAH', 'javah', ENV=ENV) @@ -883,6 +921,12 @@ class TestSCons(TestCommon): return where_javah def java_where_rmic(self, version=None): + """ + Find java rmic tool. + + :param version: if set, match only that version + :return: path to rmic + """ ENV = self.java_ENV(version) if self.detect_tool('rmic'): where_rmic = self.detect('RMIC', 'rmic', ENV=ENV) -- cgit v0.12 From 8fe02cb3bae12fefd95770a7caf411a520965ba5 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 14 Apr 2019 13:04:56 -0600 Subject: [PR #3352] fix sider complaints: bare except Sider in the CI system doesn't like the change which added a try/except block for a version without a dot. This was copied from a different file, so to fix the complaint, four instances are actually updated to "except ValueError" - I am assuming that this is the only thing that could go wrong, a version string cannot be converted with int(foo), which would throw ValueError. Signed-off-by: Mats Wichmann --- test/Java/RMIC.py | 4 ++-- test/Repository/RMIC.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Java/RMIC.py b/test/Java/RMIC.py index 021f666..c8848e9 100644 --- a/test/Java/RMIC.py +++ b/test/Java/RMIC.py @@ -102,13 +102,13 @@ if java_version.count('.') == 1: major, minor = java_version.split('.') try: curver = (int(major), int(minor)) - except: + except ValueError: pass elif java_version.count('.') == 0: # java 11? try: curver = (int(java_version), 0) - except: + except ValueError: pass # Check the version of the found Java compiler. diff --git a/test/Repository/RMIC.py b/test/Repository/RMIC.py index 3eeab9b..aa4a57e 100644 --- a/test/Repository/RMIC.py +++ b/test/Repository/RMIC.py @@ -44,13 +44,13 @@ if java_version.count('.') == 1: major, minor = java_version.split('.') try: curver = (int(major), int(minor)) - except: + except ValueError: pass elif java_version.count('.') == 0: # java 11? try: curver = (int(java_version), 0) - except: + except ValueError: pass # Check the version of the found Java compiler. -- cgit v0.12 From f9385b71f66ae48a353ccdea5b16f416f5b803a8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 16 Apr 2019 08:55:09 -0600 Subject: Add a test for JDK glob matching Signed-off-by: Mats Wichmann --- test/Java/Java-version.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 test/Java/Java-version.py diff --git a/test/Java/Java-version.py b/test/Java/Java-version.py new file mode 100644 index 0000000..3c13875 --- /dev/null +++ b/test/Java/Java-version.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test globbing to find a JDK +""" + +import os +import sys + +import TestSCons + +_python_ = TestSCons._python_ + +test = TestSCons.TestSCons() + +if sys.platform == 'win32': + test.subdir(['Program Files']) + test.subdir(['Program Files', 'Java']) + test.subdir(['Program Files', 'Java', 'jdk1.8.0_201']) + test.subdir(['Program Files', 'Java', 'jdk1.8.0_201', 'bin']) + test.subdir(['Program Files', 'Java', 'jdk-11.0.2']) + test.subdir(['Program Files', 'Java', 'jdk-11.0.2', 'bin']) + + test.write(['Program Files', 'Java', 'jdk1.8.0_201', 'bin', 'javac.exe'], "echo Java 1.8") + test.write(['Program Files', 'Java', 'jdk-11.0.2', 'bin', 'javac.exe'], "echo Java 11.0") +else: + test.subdir(['jvm']) + test.subdir(['jvm', 'java']) + test.subdir(['jvm', 'java', 'bin']) + test.subdir(['jvm', 'java-11-openjdk-11.0.2']) + test.subdir(['jvm', 'java-11-openjdk-11.0.2', 'bin']) + test.subdir(['jvm', 'java-1.8-openjdk-1.8.0']) + test.subdir(['jvm', 'java-1.8-openjdk-1.8.0', 'bin']) + + test.write(['jvm', 'java', 'bin', 'javac'], "echo Java 1.8") + test.write(['jvm', 'java-11-openjdk-11.0.2', 'bin', 'javac'], "echo Java 11.0") + test.write(['jvm', 'java-1.8-openjdk-1.8.0', 'bin', 'javac'], "echo Java 1.8") + +for version in (None, "1.8", "11.0"): + if version: + if sys.platform == 'win32': + patterns = [ + 'Program Files*/Java/jdk*%s*/bin' % version, + ] + else: + patterns = [ + 'jvm/*-%s*/bin' % version, + ] + else: + if sys.platform == 'win32': + patterns = [ + 'Program Files*/Java/jdk*/bin', + ] + else: + patterns = [ + 'jvm/*/bin', + ] + java_path = test.paths(patterns) + #print(java_path) + if not java_path: + msg = "glob pattern {%s} found no JDK matches" % patterns[0] + test.fail_test(message=msg) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From b9859740b1253ec58e5ab43ae345432dbeadd3ae Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 15 Apr 2019 08:08:53 -0600 Subject: [PR #3337] centralize definition of candidates A list of possible executable names are provided in several places in yacc and lex tools, make it a little cleaner by defining once, at the top. Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/lex.py | 19 +++++++++++-------- src/engine/SCons/Tool/yacc.py | 33 ++++++++++++--------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py index 91d50fd..6b83aa9 100644 --- a/src/engine/SCons/Tool/lex.py +++ b/src/engine/SCons/Tool/lex.py @@ -45,6 +45,11 @@ from SCons.Platform.win32 import CHOCO_DEFAULT_PATH LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR") +if sys.platform == 'win32': + BINS = ['flex', 'lex', 'win_flex'] +else: + BINS = ["flex", "lex"] + def lexEmitter(target, source, env): sourceBase, sourceExt = os.path.splitext(SCons.Util.to_String(source[0])) @@ -79,12 +84,10 @@ def get_lex_path(env, append_paths=False): :param append_paths: if set, add the path to the tool to PATH :return: path to lex tool, if found """ - bins = ['flex', 'lex', 'win_flex'] - - for prog in bins: + for prog in BINS: bin_path = SCons.Tool.find_program_path( - env, - prog, + env, + prog, default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) if bin_path: if append_paths: @@ -117,19 +120,19 @@ def generate(env): if sys.platform == 'win32': # ignore the return - we do not need the full path here _ = get_lex_path(env, append_paths=True) - env["LEX"] = env.Detect(['flex', 'lex', 'win_flex']) + env["LEX"] = env.Detect(BINS) if not env.get("LEXUNISTD"): env["LEXUNISTD"] = SCons.Util.CLVar("") env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET" else: - env["LEX"] = env.Detect(["flex", "lex"]) + env["LEX"] = env.Detect(BINS) env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" def exists(env): if sys.platform == 'win32': return get_lex_path(env) else: - return env.Detect(["flex", "lex"]) + return env.Detect(BINS) # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Tool/yacc.py b/src/engine/SCons/Tool/yacc.py index 07ba8da..2a5ffd8 100644 --- a/src/engine/SCons/Tool/yacc.py +++ b/src/engine/SCons/Tool/yacc.py @@ -45,6 +45,11 @@ from SCons.Platform.win32 import CHOCO_DEFAULT_PATH YaccAction = SCons.Action.Action("$YACCCOM", "$YACCCOMSTR") +if sys.platform == 'win32': + BINS = ['bison', 'yacc', 'win_bison'] +else: + BINS = ["bison", "yacc"] + def _yaccEmitter(target, source, env, ysuf, hsuf): yaccflags = env.subst("$YACCFLAGS", target=target, source=source) flags = SCons.Util.CLVar(yaccflags) @@ -109,12 +114,10 @@ def get_yacc_path(env, append_paths=False): :param append_paths: if set, add the path to the tool to PATH :return: path to yacc tool, if found """ - bins = ['bison', 'yacc', 'win_bison'] - - for prog in bins: + for prog in BINS: bin_path = SCons.Tool.find_program_path( - env, - prog, + env, + prog, default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) if bin_path: if append_paths: @@ -143,33 +146,21 @@ def generate(env): cxx_file.add_emitter('.yy', yyEmitter) if sys.platform == 'win32': - bison = SCons.Tool.find_program_path(env, 'bison', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) - if bison: - bison_bin_dir = os.path.dirname(bison) - env.AppendENVPath('PATH', bison_bin_dir) - else: - SCons.Warnings.Warning('yacc tool requested, but bison binary not found in ENV PATH') - - if sys.platform == 'win32': - # ignore the return - we do not need the full path here + # ignore the return, all we need is for the path to be added _ = get_yacc_path(env, append_paths=True) - env["YACC"] = env.Detect(['bison', 'yacc', 'win_bison']) - else: - env["YACC"] = env.Detect(["bison", "yacc"]) - + + env["YACC"] = env.Detect(BINS) env['YACCFLAGS'] = SCons.Util.CLVar('') env['YACCCOM'] = '$YACC $YACCFLAGS -o $TARGET $SOURCES' env['YACCHFILESUFFIX'] = '.h' - env['YACCHXXFILESUFFIX'] = '.hpp' - env['YACCVCGFILESUFFIX'] = '.vcg' def exists(env): if sys.platform == 'win32': return get_yacc_path(env) else: - return env.Detect(['bison', 'yacc']) + return env.Detect(BINS) # Local Variables: # tab-width:4 -- cgit v0.12 From 5728acaff9b6bc9466bc6f1e5ae8280964443ce1 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 17 Apr 2019 07:46:58 -0600 Subject: Move the jdk-version-glob test to unit tests Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/JavaCommonTests.py | 50 +++++++++++++++++ test/Java/Java-version.py | 94 -------------------------------- 2 files changed, 50 insertions(+), 94 deletions(-) delete mode 100644 test/Java/Java-version.py diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py index 10be671..fe04ce9 100644 --- a/src/engine/SCons/Tool/JavaCommonTests.py +++ b/src/engine/SCons/Tool/JavaCommonTests.py @@ -30,6 +30,7 @@ import unittest import SCons.Scanner.IDL import SCons.Tool.JavaCommon +import TestSCons # Adding trace=trace to any of the parse_jave() calls below will cause # the parser to spit out trace messages of the tokens it sees and the @@ -608,6 +609,55 @@ public class AnonDemo { assert expect == classes, (expect, classes) + def test_jdk_globs(self): + test = TestSCons.TestSCons(workdir='') + + if sys.platform == 'win32': + test.subdir(['Program Files'], + ['Program Files', 'Java'], + ['Program Files', 'Java', 'jdk1.8.0_201'], + ['Program Files', 'Java', 'jdk1.8.0_201', 'bin'], + ['Program Files', 'Java', 'jdk-11.0.2'], + ['Program Files', 'Java', 'jdk-11.0.2', 'bin']) + + test.write(['Program Files', 'Java', 'jdk1.8.0_201', 'bin', 'javac.exe'], "echo Java 1.8") + test.write(['Program Files', 'Java', 'jdk-11.0.2', 'bin', 'javac.exe'], "echo Java 11.0") + else: + test.subdir(['jvm'], + ['jvm', 'java'], + ['jvm', 'java', 'bin'], + ['jvm', 'java-11-openjdk-11.0.2'], + ['jvm', 'java-11-openjdk-11.0.2', 'bin'], + ['jvm', 'java-1.8-openjdk-1.8.0'], + ['jvm', 'java-1.8-openjdk-1.8.0', 'bin']) + + test.write(['jvm', 'java', 'bin', 'javac'], "echo Java 1.8") + test.write(['jvm', 'java-11-openjdk-11.0.2', 'bin', 'javac'], "echo Java 11.0") + test.write(['jvm', 'java-1.8-openjdk-1.8.0', 'bin', 'javac'], "echo Java 1.8") + + for version in (None, "1.8", "11.0"): + if version: + if sys.platform == 'win32': + patterns = [ + 'Program Files*/Java/jdk*%s*/bin' % version, + ] + else: + patterns = [ + 'jvm/*-%s*/bin' % version, + ] + else: + if sys.platform == 'win32': + patterns = [ + 'Program Files*/Java/jdk*/bin', + ] + else: + patterns = [ + 'jvm/*/bin', + ] + java_path = test.paths(patterns) + assert java_path, "no java found by pattern %s" % patterns[0] + + if __name__ == "__main__": unittest.main() diff --git a/test/Java/Java-version.py b/test/Java/Java-version.py deleted file mode 100644 index 3c13875..0000000 --- a/test/Java/Java-version.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Test globbing to find a JDK -""" - -import os -import sys - -import TestSCons - -_python_ = TestSCons._python_ - -test = TestSCons.TestSCons() - -if sys.platform == 'win32': - test.subdir(['Program Files']) - test.subdir(['Program Files', 'Java']) - test.subdir(['Program Files', 'Java', 'jdk1.8.0_201']) - test.subdir(['Program Files', 'Java', 'jdk1.8.0_201', 'bin']) - test.subdir(['Program Files', 'Java', 'jdk-11.0.2']) - test.subdir(['Program Files', 'Java', 'jdk-11.0.2', 'bin']) - - test.write(['Program Files', 'Java', 'jdk1.8.0_201', 'bin', 'javac.exe'], "echo Java 1.8") - test.write(['Program Files', 'Java', 'jdk-11.0.2', 'bin', 'javac.exe'], "echo Java 11.0") -else: - test.subdir(['jvm']) - test.subdir(['jvm', 'java']) - test.subdir(['jvm', 'java', 'bin']) - test.subdir(['jvm', 'java-11-openjdk-11.0.2']) - test.subdir(['jvm', 'java-11-openjdk-11.0.2', 'bin']) - test.subdir(['jvm', 'java-1.8-openjdk-1.8.0']) - test.subdir(['jvm', 'java-1.8-openjdk-1.8.0', 'bin']) - - test.write(['jvm', 'java', 'bin', 'javac'], "echo Java 1.8") - test.write(['jvm', 'java-11-openjdk-11.0.2', 'bin', 'javac'], "echo Java 11.0") - test.write(['jvm', 'java-1.8-openjdk-1.8.0', 'bin', 'javac'], "echo Java 1.8") - -for version in (None, "1.8", "11.0"): - if version: - if sys.platform == 'win32': - patterns = [ - 'Program Files*/Java/jdk*%s*/bin' % version, - ] - else: - patterns = [ - 'jvm/*-%s*/bin' % version, - ] - else: - if sys.platform == 'win32': - patterns = [ - 'Program Files*/Java/jdk*/bin', - ] - else: - patterns = [ - 'jvm/*/bin', - ] - java_path = test.paths(patterns) - #print(java_path) - if not java_path: - msg = "glob pattern {%s} found no JDK matches" % patterns[0] - test.fail_test(message=msg) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 43881bb8c282488f61e2c979ab4435d66390d8a6 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 17 Apr 2019 17:49:39 -0700 Subject: Issue #3350 - Refactor EnvironmentError to SConsEnvironmentError to avoid overriding python's native EnvironmentError --- src/CHANGES.txt | 4 ++++ src/engine/SCons/EnvironmentTests.py | 8 ++++---- src/engine/SCons/Errors.py | 4 ++-- src/engine/SCons/ErrorsTests.py | 4 ++-- src/engine/SCons/Tool/ToolTests.py | 2 +- src/engine/SCons/Tool/__init__.py | 12 ++++++------ src/engine/SCons/Tool/intelc.py | 2 +- src/engine/SCons/Tool/packaging/__init__.py | 2 +- test/GetBuildFailures/serial.py | 12 ++++++------ testing/framework/TestCmdTests.py | 6 +++--- 10 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 84a8056..e3b2d1b 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -7,6 +7,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER + From William Deegan: + - Fix Issue #3350 - SCons Exception EnvironmentError is conflicting with Python's EnvironmentError. + Renamed to SConsEnvironmentError + From Mats Wichmann: - scons-time takes more care closing files and uses safer mkdtemp to avoid possible races on multi-job runs. diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 2525e0f..760e8d3 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -2433,16 +2433,16 @@ f5: \ exc_caught = None try: env.Tool('does_not_exist') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: exc_caught = 1 - assert exc_caught, "did not catch expected EnvironmentError" + assert exc_caught, "did not catch expected SConsEnvironmentError" exc_caught = None try: env.Tool('$NONE') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: exc_caught = 1 - assert exc_caught, "did not catch expected EnvironmentError" + assert exc_caught, "did not catch expected SConsEnvironmentError" # Use a non-existent toolpath directory just to make sure we # can call Tool() with the keyword argument. diff --git a/src/engine/SCons/Errors.py b/src/engine/SCons/Errors.py index 3746d5d..a3a891f 100644 --- a/src/engine/SCons/Errors.py +++ b/src/engine/SCons/Errors.py @@ -124,7 +124,7 @@ class UserError(Exception): class StopError(Exception): pass -class EnvironmentError(Exception): +class SConsEnvironmentError(Exception): pass class MSVCError(IOError): @@ -184,7 +184,7 @@ def convert_to_BuildError(status, exc_info=None): filename=filename, exc_info=exc_info) - elif isinstance(status, (EnvironmentError, OSError, IOError)): + elif isinstance(status, (SConsEnvironmentError, OSError, IOError)): # If an IOError/OSError happens, raise a BuildError. # Report the name of the file or directory that caused the # error, which might be different from the target being built diff --git a/src/engine/SCons/ErrorsTests.py b/src/engine/SCons/ErrorsTests.py index 5c16160..d777ba1 100644 --- a/src/engine/SCons/ErrorsTests.py +++ b/src/engine/SCons/ErrorsTests.py @@ -101,10 +101,10 @@ class ErrorsTestCase(unittest.TestCase): assert e.node == "node" def test_convert_EnvironmentError_to_BuildError(self): - """Test the convert_to_BuildError function on EnvironmentError + """Test the convert_to_BuildError function on SConsEnvironmentError exceptions. """ - ee = SCons.Errors.EnvironmentError("test env error") + ee = SCons.Errors.SConsEnvironmentError("test env error") be = SCons.Errors.convert_to_BuildError(ee) assert be.errstr == "test env error" assert be.status == 2 diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index a4353b1..4bc5106 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -72,7 +72,7 @@ class ToolTestCase(unittest.TestCase): try: p = SCons.Tool.Tool('_does_not_exist_') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: pass else: raise diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 8fbd587..c0aa634 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -149,7 +149,7 @@ class Tool(object): except ImportError as e: splitname = self.name.split('.') if str(e) != "No module named %s" % splitname[0]: - raise SCons.Errors.EnvironmentError(e) + raise SCons.Errors.SConsEnvironmentError(e) try: import zipimport except ImportError: @@ -211,13 +211,13 @@ class Tool(object): if spec is None: error_string = "No module named %s" % self.name - raise SCons.Errors.EnvironmentError(error_string) + raise SCons.Errors.SConsEnvironmentError(error_string) module = importlib.util.module_from_spec(spec) if module is None: if debug: print("MODULE IS NONE:%s" % self.name) error_string = "No module named %s" % self.name - raise SCons.Errors.EnvironmentError(error_string) + raise SCons.Errors.SConsEnvironmentError(error_string) # Don't reload a tool we already loaded. sys_modules_value = sys.modules.get(found_name, False) @@ -258,7 +258,7 @@ class Tool(object): return module except ImportError as e: if str(e) != "No module named %s" % self.name: - raise SCons.Errors.EnvironmentError(e) + raise SCons.Errors.SConsEnvironmentError(e) try: import zipimport importer = zipimport.zipimporter(sys.modules['SCons.Tool'].__path__[0]) @@ -267,10 +267,10 @@ class Tool(object): return module except ImportError as e: m = "No tool named '%s': %s" % (self.name, e) - raise SCons.Errors.EnvironmentError(m) + raise SCons.Errors.SConsEnvironmentError(m) except ImportError as e: m = "No tool named '%s': %s" % (self.name, e) - raise SCons.Errors.EnvironmentError(m) + raise SCons.Errors.SConsEnvironmentError(m) def __call__(self, env, *args, **kw): if self.init_kw is not None: diff --git a/src/engine/SCons/Tool/intelc.py b/src/engine/SCons/Tool/intelc.py index 9fc0bf7..17e7f31 100644 --- a/src/engine/SCons/Tool/intelc.py +++ b/src/engine/SCons/Tool/intelc.py @@ -221,7 +221,7 @@ def get_all_compiler_versions(): versions = [] try: while i < 100: - subkey = SCons.Util.RegEnumKey(k, i) # raises EnvironmentError + subkey = SCons.Util.RegEnumKey(k, i) # raises SConsEnvironmentError # Check that this refers to an existing dir. # This is not 100% perfect but should catch common # installation issues like when the compiler was installed diff --git a/src/engine/SCons/Tool/packaging/__init__.py b/src/engine/SCons/Tool/packaging/__init__.py index 174ab8c..3c2c2f4 100644 --- a/src/engine/SCons/Tool/packaging/__init__.py +++ b/src/engine/SCons/Tool/packaging/__init__.py @@ -125,7 +125,7 @@ def Package(env, target=None, source=None, **kw): # the specific packager is a relative import return importlib.import_module("." + type, __name__) except ImportError as e: - raise EnvironmentError("packager %s not available: %s"%(type,str(e))) + raise SConsEnvironmentError("packager %s not available: %s" % (type, str(e))) packagers = list(map(load_packager, PACKAGETYPE)) diff --git a/test/GetBuildFailures/serial.py b/test/GetBuildFailures/serial.py index 144d8bc..4d1b7cd 100644 --- a/test/GetBuildFailures/serial.py +++ b/test/GetBuildFailures/serial.py @@ -83,8 +83,8 @@ Command('f08', 'f08.in', raiseExcAction(SCons.Errors.UserError("My User Error")) Command('f09', 'f09.in', returnExcAction(SCons.Errors.UserError("My User Error"))) Command('f10', 'f10.in', raiseExcAction(MyBuildError(errstr="My Build Error", status=7))) Command('f11', 'f11.in', returnExcAction(MyBuildError(errstr="My Build Error", status=7))) -Command('f12', 'f12.in', raiseExcAction(OSError(123, "My EnvironmentError", "f12"))) -Command('f13', 'f13.in', returnExcAction(OSError(123, "My EnvironmentError", "f13"))) +Command('f12', 'f12.in', raiseExcAction(OSError(123, "My SConsEnvironmentError", "f12"))) +Command('f13', 'f13.in', returnExcAction(OSError(123, "My SConsEnvironmentError", "f13"))) Command('f14', 'f14.in', raiseExcAction(SCons.Errors.InternalError("My InternalError"))) Command('f15', 'f15.in', returnExcAction(SCons.Errors.InternalError("My InternalError"))) @@ -173,9 +173,9 @@ BF: f10 failed (7): My Build Error BF: action(["f10"], ["f10.in"]) BF: f11 failed (7): My Build Error BF: action(["f11"], ["f11.in"]) -BF: f12 failed (123): My EnvironmentError +BF: f12 failed (123): My SConsEnvironmentError BF: action(["f12"], ["f12.in"]) -BF: f13 failed (123): My EnvironmentError +BF: f13 failed (123): My SConsEnvironmentError BF: action(["f13"], ["f13.in"]) BF: f14 failed (2): InternalError : My InternalError BF: action(["f14"], ["f14.in"]) @@ -191,8 +191,8 @@ scons: *** [f08] My User Error scons: *** [f09] My User Error scons: *** [f10] My Build Error scons: *** [f11] My Build Error -scons: *** [f12] f12: My EnvironmentError -scons: *** [f13] f13: My EnvironmentError +scons: *** [f12] f12: My SConsEnvironmentError +scons: *** [f13] f13: My SConsEnvironmentError scons: *** [f14] InternalError : My InternalError """) + \ """\ diff --git a/testing/framework/TestCmdTests.py b/testing/framework/TestCmdTests.py index 0b6aab9..d6922d9 100644 --- a/testing/framework/TestCmdTests.py +++ b/testing/framework/TestCmdTests.py @@ -1634,7 +1634,7 @@ class rmdir_TestCase(TestCmdTestCase): except EnvironmentError: pass else: - raise Exception("did not catch expected EnvironmentError") + raise Exception("did not catch expected SConsEnvironmentError") test.subdir(['sub'], ['sub', 'dir'], @@ -1649,7 +1649,7 @@ class rmdir_TestCase(TestCmdTestCase): except EnvironmentError: pass else: - raise Exception("did not catch expected EnvironmentError") + raise Exception("did not catch expected SConsEnvironmentError") assert os.path.isdir(s_d_o), "%s is gone?" % s_d_o @@ -1658,7 +1658,7 @@ class rmdir_TestCase(TestCmdTestCase): except EnvironmentError: pass else: - raise Exception("did not catch expected EnvironmentError") + raise Exception("did not catch expected SConsEnvironmentError") assert os.path.isdir(s_d_o), "%s is gone?" % s_d_o -- cgit v0.12 From 1a98847cf341370f2bc9224d4e43ce62e679d74a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 19 Apr 2019 22:02:13 -0400 Subject: Issue #3350 - Add TEMPFILEARGJOINBYTE as a variable to be used to join each item placed into a TEMPFILE. Previously hardcoded as a bytearray space, now it is overridden by msvc and mslink tools --- src/engine/SCons/Defaults.py | 1 + src/engine/SCons/Platform/__init__.py | 3 ++- src/engine/SCons/Tool/mslink.py | 6 ++++++ src/engine/SCons/Tool/msvc.py | 6 ++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index 479ef7e..e01b8a0 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -582,6 +582,7 @@ ConstructionEnvironment = { '__DSHLIBVERSIONFLAGS' : '${__libversionflags(__env__,"DSHLIBVERSION","_DSHLIBVERSIONFLAGS")}', 'TEMPFILE' : NullCmdGenerator, + 'TEMPFILEARGJOINBYTE': bytearray(' '), 'Dir' : Variable_Method_Caller('TARGET', 'Dir'), 'Dirs' : Variable_Method_Caller('TARGET', 'Dirs'), 'File' : Variable_Method_Caller('TARGET', 'File'), diff --git a/src/engine/SCons/Platform/__init__.py b/src/engine/SCons/Platform/__init__.py index e06eb0f..068ab89 100644 --- a/src/engine/SCons/Platform/__init__.py +++ b/src/engine/SCons/Platform/__init__.py @@ -226,7 +226,8 @@ class TempFileMunge(object): prefix = '@' args = list(map(SCons.Subst.quote_spaces, cmd[1:])) - os.write(fd, bytearray(" ".join(args) + "\n",'utf-8')) + join_char = env.subst('TEMPFILEARGJOINBYTE') + os.write(fd, bytearray(join_char.join(args) + "\n",'utf-8')) os.close(fd) # XXX Using the SCons.Action.print_actions value directly # like this is bogus, but expedient. This class should diff --git a/src/engine/SCons/Tool/mslink.py b/src/engine/SCons/Tool/mslink.py index c8b00d2..176eb79 100644 --- a/src/engine/SCons/Tool/mslink.py +++ b/src/engine/SCons/Tool/mslink.py @@ -327,6 +327,12 @@ def generate(env): env['LDMODULEEMITTER'] = [ldmodEmitter] env['LDMODULECOM'] = compositeLdmodAction + # Issue #3350 + # Change tempfile argument joining character from a bytearray space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOINBYTE'] = bytearray('\n') + def exists(env): return msvc_exists(env) diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index 6cfa245..91ffa33 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -283,6 +283,12 @@ def generate(env): msvc_set_PCHPDBFLAGS(env) + # Issue #3350 + # Change tempfile argument joining character from a bytearray space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOINBYTE'] = bytearray('\n') + env['PCHCOM'] = '$CXX /Fo${TARGETS[1]} $CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS /c $SOURCES /Yc$PCHSTOP /Fp${TARGETS[0]} $CCPDBFLAGS $PCHPDBFLAGS' env['BUILDERS']['PCH'] = pch_builder -- cgit v0.12 From 54975192faaf0042636ff5b1fc3617dc834294f4 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 21 Apr 2019 10:40:07 -0600 Subject: Fixup some code triggering pylint errors. Assorted fixups: exception types, redefined functions, globals, etc. Some old code removed to resolve issues (hashlib is always present on modern Pythons; no longer need the code for 2.5-and-earlier optparse). cmp is not a builtin function in Py3, drop one (unused) use; replace one. Fix another instance of renaming to SConsEnvironmentError. TODO flagged some instances of doing a raise without argument but not inside a try block - this is not considered legal, since raise with no argument is for re-raising an exception, but I don't know exactly how to resolve this in these cases. Also flagged an instance of raising an int instead of an exception class. We can either leave these as markers or update the PR. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 6 ++ src/engine/SCons/ActionTests.py | 8 +-- src/engine/SCons/BuilderTests.py | 26 +++---- src/engine/SCons/CacheDir.py | 12 ++-- src/engine/SCons/DefaultsTests.py | 6 +- src/engine/SCons/EnvironmentTests.py | 40 +++++------ src/engine/SCons/Executor.py | 16 ++--- src/engine/SCons/JobTests.py | 4 +- src/engine/SCons/Node/FSTests.py | 8 +-- src/engine/SCons/Node/NodeTests.py | 4 +- src/engine/SCons/Node/__init__.py | 4 +- src/engine/SCons/Platform/PlatformTests.py | 2 +- src/engine/SCons/SConf.py | 1 + src/engine/SCons/Script/Main.py | 32 ++++----- src/engine/SCons/Script/SConsOptions.py | 45 ++---------- src/engine/SCons/SubstTests.py | 3 +- src/engine/SCons/Taskmaster.py | 8 +-- src/engine/SCons/Tool/MSCommon/netframework.py | 2 +- src/engine/SCons/Tool/ToolTests.py | 4 +- src/engine/SCons/Tool/intelc.py | 11 +-- src/engine/SCons/Tool/packaging/__init__.py | 4 +- src/engine/SCons/Tool/packaging/ipk.py | 14 ++-- src/engine/SCons/Tool/xgettext.py | 53 +++++---------- src/engine/SCons/Util.py | 94 ++++++++++++++------------ src/engine/SCons/compat/__init__.py | 10 +-- src/engine/SCons/cpp.py | 2 +- 26 files changed, 187 insertions(+), 232 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index e3b2d1b..86e98ac 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -17,6 +17,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Use importlib to dynamically load tool and platform modules instead of imp module - sconsign: default to .sconsign.dblite if no filename is specified. Be more informative in case of unsupported pickle protocol (py2 only). + - assorted fixups for pylint: exception types, redefined functions, + globals, etc. Some old code removed to resolve issues (hashlib is + always present on modern Pythons; no longer need the code for + 2.5-and-earlier optparse). cmp is not a builtin function in Py3, + drop one (unused) use; replace one. Fix another instance of + renaming to SConsEnvironmentError. From John Doe: diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index efe6e98..042d3dd 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -1554,19 +1554,19 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + repr( func_matches[sys.version_info[:2]]) - def f_global(target, source, env, for_signature): + def f_global2(target, source, env, for_signature): return SCons.Action.Action(GlobalFunc, varlist=['XYZ']) - def f_local(target, source, env, for_signature): + def f_local2(target, source, env, for_signature): return SCons.Action.Action(LocalFunc, varlist=['XYZ']) matches_foo = func_matches[sys.version_info[:2]] + b'foo' - a = self.factory(f_global) + a = self.factory(f_global2) c = a.get_contents(target=[], source=[], env=env) assert c in matches_foo, repr(c) - a = self.factory(f_local) + a = self.factory(f_local2) c = a.get_contents(target=[], source=[], env=env) assert c in matches_foo, repr(c) diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py index 847e30a..b03a425 100644 --- a/src/engine/SCons/BuilderTests.py +++ b/src/engine/SCons/BuilderTests.py @@ -409,10 +409,6 @@ class BuilderTestCase(unittest.TestCase): builder = SCons.Builder.Builder(generator=generator) assert builder.action.generator == generator - def test_get_name(self): - """Test the get_name() method - """ - def test_cmp(self): """Test simple comparisons of Builder objects """ @@ -552,7 +548,7 @@ class BuilderTestCase(unittest.TestCase): def test_src_suffix(self): """Test Builder creation with a specified source file suffix - + Make sure that the '.' separator is appended to the beginning if it isn't already present. """ @@ -685,7 +681,7 @@ class BuilderTestCase(unittest.TestCase): pass if (len(source) == 1 and len(target) == 1): env['CNT'][0] = env['CNT'][0] + 1 - + env = Environment() infiles = [] outfiles = [] @@ -732,8 +728,8 @@ class BuilderTestCase(unittest.TestCase): pass else: assert 0 - - + + def test_lists(self): """Testing handling lists of targets and source""" def function2(target, source, env, tlist = [outfile, outfile2], **kw): @@ -881,7 +877,7 @@ class BuilderTestCase(unittest.TestCase): def func(self): pass - + scanner = SCons.Scanner.Base(func, name='fooscan') b1 = SCons.Builder.Builder(action='bld', target_scanner=scanner) @@ -890,8 +886,8 @@ class BuilderTestCase(unittest.TestCase): assert b1 == b2 assert b1 != b3 - - def test_src_scanner(slf): + + def test_src_scanner(self): """Testing ability to set a source file scanner through a builder.""" class TestScanner(object): def key(self, env): @@ -1245,7 +1241,7 @@ class BuilderTestCase(unittest.TestCase): for t in target: t.builder = nb return [nn], source - + builder=SCons.Builder.Builder(action='foo', emitter=emit, target_factory=MyNode, @@ -1321,7 +1317,7 @@ class BuilderTestCase(unittest.TestCase): builder2 = SCons.Builder.Builder(action='foo', emitter='$EMITTERLIST', node_factory=MyNode) - + env = Environment(EMITTERLIST = [emit2a, emit2b]) tgts = builder2(env, target='target-2', source='aaa.2') @@ -1419,7 +1415,7 @@ class BuilderTestCase(unittest.TestCase): b6 = SCons.Builder.Builder(action='foo') assert isinstance(b4, SCons.Builder.CompositeBuilder) assert isinstance(b4.action, SCons.Action.CommandGeneratorAction) - + env = Environment(BUILDERS={'bldr1': b1, 'bldr2': b2, 'bldr3': b3, @@ -1481,7 +1477,7 @@ class CompositeBuilderTestCase(unittest.TestCase): tgt = builder(env, source=[]) assert tgt == [], tgt - + assert isinstance(builder, SCons.Builder.CompositeBuilder) assert isinstance(builder.action, SCons.Action.CommandGeneratorAction) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index ab23f31..3d8e7bd 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -111,7 +111,7 @@ def CachePushFunc(target, source, env): # has beaten us creating the directory. if not fs.isdir(cachedir): msg = errfmt % (str(target), cachefile) - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) try: if fs.islink(t.get_internal_path()): @@ -177,8 +177,8 @@ class CacheDir(object): # If someone else is trying to create the directory at # the same time as me, bad things will happen msg = "Failed to create cache directory " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + self.config['prefix_len'] = 2 if not os.path.exists(config_file): try: @@ -186,15 +186,15 @@ class CacheDir(object): json.dump(self.config, config) except: msg = "Failed to write cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) else: try: with open(config_file) as config: self.config = json.load(config) except ValueError: msg = "Failed to read cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/engine/SCons/DefaultsTests.py b/src/engine/SCons/DefaultsTests.py index e04d1eb..2cbad70 100644 --- a/src/engine/SCons/DefaultsTests.py +++ b/src/engine/SCons/DefaultsTests.py @@ -69,11 +69,11 @@ class DefaultsTestCase(unittest.TestCase): test.write(file, "test\n") try: mkdir_func(file) - except os.error as e: + except OSError as e: pass else: - fail("expected os.error") - + self.fail("expected OSError") + if __name__ == "__main__": unittest.main() diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 760e8d3..347c410 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -74,7 +74,7 @@ def diff_dict(d1, d2): s2 = s2 + " " + repr(k) + " : " + repr(d2[k]) + "\n" else: s1 = s1 + " " + repr(k) + " : " + repr(d1[k]) + "\n" - elif k in env2: + elif k in d2: s2 = s2 + " " + repr(k) + " : " + repr(d2[k]) + "\n" s1 = s1 + "}\n" s2 = s2 + "}\n" @@ -263,39 +263,39 @@ class SubstitutionTestCase(unittest.TestCase): nodes = env.arg2nodes("Util.py UtilTests.py", Factory) assert len(nodes) == 1, nodes assert isinstance(nodes[0], X) - assert nodes[0].name == "Util.py UtilTests.py" + assert nodes[0].name == "Util.py UtilTests.py", nodes[0].name nodes = env.arg2nodes(u"Util.py UtilTests.py", Factory) assert len(nodes) == 1, nodes assert isinstance(nodes[0], X) - assert nodes[0].name == u"Util.py UtilTests.py" + assert nodes[0].name == u"Util.py UtilTests.py", nodes[0].name nodes = env.arg2nodes(["Util.py", "UtilTests.py"], Factory) assert len(nodes) == 2, nodes assert isinstance(nodes[0], X) assert isinstance(nodes[1], X) - assert nodes[0].name == "Util.py" - assert nodes[1].name == "UtilTests.py" + assert nodes[0].name == "Util.py", nodes[0].name + assert nodes[1].name == "UtilTests.py", nodes[1].name n1 = Factory("Util.py") nodes = env.arg2nodes([n1, "UtilTests.py"], Factory) assert len(nodes) == 2, nodes assert isinstance(nodes[0], X) assert isinstance(nodes[1], X) - assert nodes[0].name == "Util.py" - assert nodes[1].name == "UtilTests.py" + assert nodes[0].name == "Util.py", nodes[0].name + assert nodes[1].name == "UtilTests.py", nodes[1].name class SConsNode(SCons.Node.Node): pass nodes = env.arg2nodes(SConsNode()) assert len(nodes) == 1, nodes - assert isinstance(nodes[0], SConsNode), node + assert isinstance(nodes[0], SConsNode), nodes[0] class OtherNode(object): pass nodes = env.arg2nodes(OtherNode()) assert len(nodes) == 1, nodes - assert isinstance(nodes[0], OtherNode), node + assert isinstance(nodes[0], OtherNode), nodes[0] def lookup_a(str, F=Factory): if str[0] == 'a': @@ -484,7 +484,7 @@ class SubstitutionTestCase(unittest.TestCase): env = SubstitutionEnvironment(AAA = '$BBB', BBB = '$CCC', CCC = 'c') l = env.subst_list("$AAA ${AAA}A ${AAA}B $BBB") - assert l == [["c", "cA", "cB", "c"]], mystr + assert l == [["c", "cA", "cB", "c"]], l env = SubstitutionEnvironment(AAA = '$BBB', BBB = '$CCC', CCC = [ 'a', 'b\nc' ]) lst = env.subst_list([ "$AAA", "B $CCC" ]) @@ -1195,7 +1195,7 @@ env4.builder1.env, env3) test_it('foo.bar') test_it('foo-bar') - def test_autogenerate(dict): + def test_autogenerate(self): """Test autogenerating variables in a dictionary.""" drive, p = os.path.splitdrive(os.getcwd()) @@ -1206,9 +1206,9 @@ env4.builder1.env, env3) drive, path = os.path.splitdrive(path) return drive.lower() + path - env = dict.TestEnvironment(LIBS = [ 'foo', 'bar', 'baz' ], - LIBLINKPREFIX = 'foo', - LIBLINKSUFFIX = 'bar') + env = self.TestEnvironment(LIBS = [ 'foo', 'bar', 'baz' ], + LIBLINKPREFIX = 'foo', + LIBLINKSUFFIX = 'bar') def RDirs(pathlist, fs=env.fs): return fs.Dir('xx').Rfindalldirs(pathlist) @@ -3272,11 +3272,11 @@ def generate(env): s = e.src_builder() assert s is None, s - def test_SourceSignatures(type): + def test_SourceSignatures(self): """Test the SourceSignatures() method""" import SCons.Errors - env = type.TestEnvironment(M = 'MD5', T = 'timestamp') + env = self.TestEnvironment(M = 'MD5', T = 'timestamp') exc_caught = None try: @@ -3312,7 +3312,7 @@ def generate(env): def test_Split(self): """Test the Split() method""" - env = self.TestEnvironment(FOO='fff', BAR='bbb') + env = self.TestEnvironment(FOO = 'fff', BAR = 'bbb') s = env.Split("foo bar") assert s == ["foo", "bar"], s s = env.Split("$FOO bar") @@ -3326,11 +3326,11 @@ def generate(env): s = env.Split("$FOO$BAR") assert s == ["fffbbb"], s - def test_TargetSignatures(type): + def test_TargetSignatures(self): """Test the TargetSignatures() method""" import SCons.Errors - env = type.TestEnvironment(B = 'build', C = 'content') + env = self.TestEnvironment(B='build', C='content') exc_caught = None try: @@ -3397,7 +3397,7 @@ def generate(env): - def test_Environment_global_variable(type): + def test_Environment_global_variable(self): """Test setting Environment variable to an Environment.Base subclass""" class MyEnv(SCons.Environment.Base): def xxx(self, string): diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 01d01cd..6c68e09 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -41,10 +41,10 @@ from SCons.compat import with_metaclass, NoSlotsPyPy class Batch(object): """Remembers exact association between targets and sources of executor.""" - + __slots__ = ('targets', 'sources') - + def __init__(self, targets=[], sources=[]): self.targets = targets self.sources = sources @@ -127,13 +127,13 @@ def execute_action_list(obj, target, kw): status = act(*args, **kw) if isinstance(status, SCons.Errors.BuildError): status.executor = obj - raise status + raise status # TODO pylint E0702: raising int not allowed elif status: msg = "Error %s" % status raise SCons.Errors.BuildError( - errstr=msg, + errstr=msg, node=obj.batches[0].targets, - executor=obj, + executor=obj, action=act) return status @@ -597,7 +597,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): disassociate Builders from Nodes entirely, so we're not going to worry about unit tests for this--at least for now. """ - + __slots__ = ('pre_actions', 'post_actions', 'env', @@ -613,7 +613,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): 'action_list', '_do_execute', '_execute_str') - + def __init__(self, *args, **kw): if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') self.batches = [Batch(kw['targets'][:], [])] @@ -649,7 +649,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): """Morph this Null executor to a real Executor object.""" batches = self.batches self.__class__ = Executor - self.__init__([]) + self.__init__([]) self.batches = batches # The following methods require morphing this Null Executor to a diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 325c0e1..26e3d37 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -73,7 +73,7 @@ class DummyLock(object): def release(self): pass -class NoThreadsException(object): +class NoThreadsException(Exception): "raised by the ParallelTestCase if threads are not supported" def __str__(self): @@ -206,7 +206,7 @@ class Taskmaster(object): self.parallel_list = [0] * (n+1) self.found_parallel = False self.Task = Task - + # 'guard' guards 'task_begin_list' and 'task_end_list' try: import threading diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 23ec48e..d09e6da 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -1072,7 +1072,7 @@ class FSTestCase(_tempdirTestCase): def Dir_test(lpath, path_, abspath_, up_path_, sep=sep, func=_do_Dir_test): return func(lpath, path_, abspath_, up_path_, sep) - + Dir_test('/', '/', '/', '/') Dir_test('', './', sub_dir, sub) Dir_test('foo', 'foo/', sub_dir_foo, './') @@ -1834,7 +1834,7 @@ class FSTestCase(_tempdirTestCase): # Should be a normalized Windows UNC path as below. assert str(f) == r'\\servername\C$\foo', \ 'UNC path %s got looked up as %s'%(path, f) - + def test_unc_drive_letter(self): """Test drive-letter lookup for windows UNC-style directories""" if sys.platform not in ('win32',): @@ -3534,11 +3534,11 @@ class clearTestCase(unittest.TestCase): e = fs.Entry('e') assert not e.exists() assert not e.rexists() - assert str(e) == 'e', str(d) + assert str(e) == 'e', str(e) e.clear() assert not e.exists() assert not e.rexists() - assert str(e) == 'e', str(d) + assert str(e) == 'e', str(e) d = fs.Dir(test.workpath('d')) test.subdir('d') diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 7d347ac..678e03e 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -271,7 +271,7 @@ class NodeInfoBaseTestCase(unittest.TestCase): f = ni1.format() assert f == ['x', 'y', 'z'], f - + field_list = ['xxx', 'zzz', 'aaa'] f = ni1.format(field_list) @@ -751,7 +751,7 @@ class NodeTestCase(unittest.TestCase): e = node.exists() assert e == 1, e - def test_exists(self): + def test_exists_repo(self): """Test evaluating whether a Node exists locally or in a repository. """ node = SCons.Node.Node() diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 131953b..b9aca96 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -103,9 +103,9 @@ implicit_deps_changed = 0 # A variable that can be set to an interface-specific function be called # to annotate a Node with information about its creation. -def do_nothing(node): pass +def do_nothing_node(node): pass -Annotate = do_nothing +Annotate = do_nothing_node # Gets set to 'True' if we're running in interactive mode. Is # currently used to release parts of a target's info during diff --git a/src/engine/SCons/Platform/PlatformTests.py b/src/engine/SCons/Platform/PlatformTests.py index 3f42eae..d157c3f 100644 --- a/src/engine/SCons/Platform/PlatformTests.py +++ b/src/engine/SCons/Platform/PlatformTests.py @@ -110,7 +110,7 @@ class PlatformTestCase(unittest.TestCase): p = SCons.Platform.Platform('_does_not_exist_') except SCons.Errors.UserError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise env = Environment() diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index b123c11..c3f93db 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -247,6 +247,7 @@ class SConfBuildTask(SCons.Taskmaster.AlwaysTask): # ConfigureCacheError and if yes, reraise the exception exc_type = self.exc_info()[0] if issubclass(exc_type, SConfError): + # TODO pylint E0704: bare raise not inside except raise elif issubclass(exc_type, SCons.Errors.BuildError): # we ignore Build Errors (occurs, when a test doesn't pass) diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index f3475f2..fdcd252 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -68,6 +68,20 @@ import SCons.Warnings import SCons.Script.Interactive +# Global variables +first_command_start = None +last_command_end = None +print_objects = 0 +print_memoizer = 0 +print_stacktrace = 0 +print_time = 0 +sconscript_time = 0 +cumulative_command_time = 0 +exit_status = 0 # final exit status, assume success by default +this_build_status = 0 # "exit status" of an individual build +num_jobs = None +delayed_warnings = [] + def fetch_win32_parallel_msg(): # A subsidiary function that exists solely to isolate this import @@ -87,15 +101,14 @@ def revert_io(): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ + class SConsPrintHelpException(Exception): pass + display = SCons.Util.display progress_display = SCons.Util.DisplayEngine() -first_command_start = None -last_command_end = None - class Progressor(object): prev = '' @@ -443,19 +456,6 @@ def python_version_deprecated(version=sys.version_info): return version < deprecated_python_version -# Global variables - -print_objects = 0 -print_memoizer = 0 -print_stacktrace = 0 -print_time = 0 -sconscript_time = 0 -cumulative_command_time = 0 -exit_status = 0 # final exit status, assume success by default -this_build_status = 0 # "exit status" of an individual build -num_jobs = None -delayed_warnings = [] - class FakeOptionParser(object): """ A do-nothing option parser, used for the initial OptionsParser variable. diff --git a/src/engine/SCons/Script/SConsOptions.py b/src/engine/SCons/Script/SConsOptions.py index 37dd644..01d365d 100644 --- a/src/engine/SCons/Script/SConsOptions.py +++ b/src/engine/SCons/Script/SConsOptions.py @@ -226,39 +226,8 @@ class SConsOption(optparse.Option): fmt = "option %s: nargs='?' is incompatible with short options" raise SCons.Errors.UserError(fmt % self._short_opts[0]) - try: - _orig_CONST_ACTIONS = optparse.Option.CONST_ACTIONS - - _orig_CHECK_METHODS = optparse.Option.CHECK_METHODS - - except AttributeError: - # optparse.Option had no CONST_ACTIONS before Python 2.5. - - _orig_CONST_ACTIONS = ("store_const",) - - def _check_const(self): - if self.action not in self.CONST_ACTIONS and self.const is not None: - raise OptionError( - "'const' must not be supplied for action %r" % self.action, - self) - - # optparse.Option collects its list of unbound check functions - # up front. This sucks because it means we can't just override - # the _check_const() function like a normal method, we have to - # actually replace it in the list. This seems to be the most - # straightforward way to do that. - - _orig_CHECK_METHODS = [optparse.Option._check_action, - optparse.Option._check_type, - optparse.Option._check_choice, - optparse.Option._check_dest, - _check_const, - optparse.Option._check_nargs, - optparse.Option._check_callback] - - CHECK_METHODS = _orig_CHECK_METHODS + [_check_nargs_optional] - - CONST_ACTIONS = _orig_CONST_ACTIONS + optparse.Option.TYPED_ACTIONS + CHECK_METHODS = optparse.Option.CHECK_METHODS + [_check_nargs_optional] + CONST_ACTIONS = optparse.Option.CONST_ACTIONS + optparse.Option.TYPED_ACTIONS class SConsOptionGroup(optparse.OptionGroup): """ @@ -364,7 +333,7 @@ class SConsOptionParser(optparse.OptionParser): in self.largs, so that any value overridden on the command line is immediately available if the user turns around and does a GetOption() right away. - + We mimic the processing of the single args in the original OptionParser._process_args(), but here we allow exact matches for long-opts only (no partial @@ -375,7 +344,7 @@ class SConsOptionParser(optparse.OptionParser): command-line arguments that 1. haven't been processed so far (self.largs), but 2. are possibly not added to the list of options yet. - + So, when we only have a value for "--myargument" yet, a command-line argument of "--myarg=test" would set it. Responsible for this behaviour is the method @@ -384,7 +353,7 @@ class SConsOptionParser(optparse.OptionParser): be unique. This would lead to further confusion, because we might want to add another option "--myarg" later on (see issue #2929). - + """ rargs = [] largs_restore = [] @@ -401,7 +370,7 @@ class SConsOptionParser(optparse.OptionParser): if "=" in l: # Split into option and value lopt = l.split("=", 1) - + if lopt[0] in self._long_opt: # Argument is already known rargs.append('='.join(lopt)) @@ -416,7 +385,7 @@ class SConsOptionParser(optparse.OptionParser): skip = True else: rargs.append(l) - + # Parse the filtered list self.parse_args(rargs, self.values) # Restore the list of remaining arguments for the diff --git a/src/engine/SCons/SubstTests.py b/src/engine/SCons/SubstTests.py index a111e4b..f6fe1ec 100644 --- a/src/engine/SCons/SubstTests.py +++ b/src/engine/SCons/SubstTests.py @@ -132,6 +132,7 @@ class SubstTestCase(unittest.TestCase): def __str__(self): return self.value + # only use of this is currently commented out below def function_foo(arg): pass @@ -1046,7 +1047,7 @@ class scons_subst_list_TestCase(SubstTestCase): node = scons_subst_list("$NODE", env, mode=SUBST_SIG, conv=s, gvars=gvars) assert node == [[n1]], node - def test_subst_list_overriding_gvars(self): + def test_subst_list_overriding_gvars2(self): """Test scons_subst_list(): supplying an overriding gvars dictionary""" env = DummyEnv({'XXX' : 'xxx'}) result = scons_subst_list('$XXX', env, gvars=env.Dictionary()) diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 1522ca2..4892026 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -27,10 +27,10 @@ import sys __doc__ = """ Generic Taskmaster module for the SCons build engine. ===================================================== - + This module contains the primary interface(s) between a wrapping user interface and the SCons build engine. There are two key classes here: - + Taskmaster ---------- This is the main engine for walking the dependency graph and @@ -477,7 +477,7 @@ class Task(object): for s in t.side_effects: if s.get_state() == NODE_EXECUTING: s.set_state(NODE_NO_STATE) - + # The side-effects may have been transferred to # NODE_NO_STATE by executed_with{,out}_callbacks, but was # not taken out of the waiting parents/pending children @@ -551,7 +551,7 @@ class Task(object): try: exc_type, exc_value, exc_traceback = exc except ValueError: - exc_type, exc_value = exc + exc_type, exc_value = exc # pylint: disable=unbalanced-tuple-unpacking exc_traceback = None # raise exc_type(exc_value).with_traceback(exc_traceback) diff --git a/src/engine/SCons/Tool/MSCommon/netframework.py b/src/engine/SCons/Tool/MSCommon/netframework.py index 787d008..b40576a 100644 --- a/src/engine/SCons/Tool/MSCommon/netframework.py +++ b/src/engine/SCons/Tool/MSCommon/netframework.py @@ -68,7 +68,7 @@ def query_versions(): # sequence comparison in python is lexicographical # which is exactly what we want. # Note we sort backwards so the highest version is first. - return cmp(bbl,aal) + return (aal > bbl) - (aal < bbl) versions.sort(versrt) else: diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index 4bc5106..f005143 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -67,14 +67,14 @@ class ToolTestCase(unittest.TestCase): SCons.Tool.Tool() except TypeError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise try: p = SCons.Tool.Tool('_does_not_exist_') except SCons.Errors.SConsEnvironmentError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise diff --git a/src/engine/SCons/Tool/intelc.py b/src/engine/SCons/Tool/intelc.py index 17e7f31..1f3fcea 100644 --- a/src/engine/SCons/Tool/intelc.py +++ b/src/engine/SCons/Tool/intelc.py @@ -37,7 +37,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import math, sys, os.path, glob, string, re is_windows = sys.platform == 'win32' -is_win64 = is_windows and (os.environ['PROCESSOR_ARCHITECTURE'] == 'AMD64' or +is_win64 = is_windows and (os.environ['PROCESSOR_ARCHITECTURE'] == 'AMD64' or ('PROCESSOR_ARCHITEW6432' in os.environ and os.environ['PROCESSOR_ARCHITEW6432'] == 'AMD64')) is_linux = sys.platform.startswith('linux') @@ -113,11 +113,6 @@ def check_abi(abi): (abi, list(valid_abis.keys()))) return abi -def vercmp(a, b): - """Compare strings as floats, - but Intel changed Linux naming convention at 9.0""" - return cmp(linux_ver_normalize(b), linux_ver_normalize(a)) - def get_version_from_list(v, vlist): """See if we can match v (string) in vlist (list of strings) Linux has to match in a fuzzy way.""" @@ -293,7 +288,7 @@ def get_all_compiler_versions(): m = re.search(r'([0-9]{0,4})(?:_sp\d*)?\.([0-9][0-9.]*)$', d) if m: versions.append("%s.%s"%(m.group(1), m.group(2))) - + def keyfunc(str): """Given a dot-separated version string, return a tuple of ints representing it.""" return [int(x) for x in str.split('.')] @@ -383,7 +378,7 @@ def get_intel_compiler_top(version, abi): top = d break return top - + top = find_in_2016style_dir(version) or find_in_2011style_dir(version) or find_in_2010style_dir(version) or find_in_2008style_dir(version) # print "INTELC: top=",top if not top: diff --git a/src/engine/SCons/Tool/packaging/__init__.py b/src/engine/SCons/Tool/packaging/__init__.py index 3c2c2f4..5795396 100644 --- a/src/engine/SCons/Tool/packaging/__init__.py +++ b/src/engine/SCons/Tool/packaging/__init__.py @@ -38,8 +38,8 @@ import os import importlib __all__ = [ - 'src_targz', 'src_tarbz2', 'src_xz', 'src_zip', - 'targz', 'tarbz2', 'xz', 'zip', + 'src_targz', 'src_tarbz2', 'src_tarxz', 'src_zip', + 'targz', 'tarbz2', 'tarxz', 'zip', 'rpm', 'msi', 'ipk', ] diff --git a/src/engine/SCons/Tool/packaging/ipk.py b/src/engine/SCons/Tool/packaging/ipk.py index 2ecaa9b..fe3f49b 100644 --- a/src/engine/SCons/Tool/packaging/ipk.py +++ b/src/engine/SCons/Tool/packaging/ipk.py @@ -3,7 +3,7 @@ # # __COPYRIGHT__ -# +# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -117,17 +117,17 @@ def build_specfiles(source, target, env): # # opened_files={} - def open_file(needle, haystack): + def open_file(needle, haystack=None): try: return opened_files[needle] except KeyError: files = filter(lambda x: x.get_path().rfind(needle) != -1, haystack) # Py3: filter returns an iterable, not a list file = list(files)[0] - opened_files[needle]=open(file.get_abspath(), 'w') + opened_files[needle] = open(file.get_abspath(), 'w') return opened_files[needle] - control_file=open_file('control', target) + control_file = open_file('control', target) if 'X_IPK_DESCRIPTION' not in env: env['X_IPK_DESCRIPTION']="%s\n %s"%(env['SUMMARY'], @@ -149,7 +149,7 @@ Description: $X_IPK_DESCRIPTION control_file.write(env.subst(content)) # - # now handle the various other files, which purpose it is to set post-, + # now handle the various other files, which purpose it is to set post-, # pre-scripts and mark files as config files. # # We do so by filtering the source files for files which are marked with @@ -161,14 +161,14 @@ Description: $X_IPK_DESCRIPTION # into the same named file. # for f in [x for x in source if 'PACKAGING_CONFIG' in dir(x)]: - config=open_file('conffiles') + config = open_file('conffiles') config.write(f.PACKAGING_INSTALL_LOCATION) config.write('\n') for str in 'POSTRM PRERM POSTINST PREINST'.split(): name="PACKAGING_X_IPK_%s"%str for f in [x for x in source if name in dir(x)]: - file=open_file(name) + file = open_file(name) file.write(env[str]) # diff --git a/src/engine/SCons/Tool/xgettext.py b/src/engine/SCons/Tool/xgettext.py index e4a17ca..936924b 100644 --- a/src/engine/SCons/Tool/xgettext.py +++ b/src/engine/SCons/Tool/xgettext.py @@ -26,6 +26,23 @@ Tool specific initialization of `xgettext` tool. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os +import re +import subprocess +import sys + +import SCons.Action +import SCons.Node.FS +import SCons.Tool +import SCons.Util +from SCons.Builder import BuilderBase +from SCons.Environment import _null +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Tool.GettextCommon import _POTargetFactory +from SCons.Tool.GettextCommon import RPaths, _detect_xgettext +from SCons.Tool.GettextCommon import _xgettext_exists + ############################################################################# class _CmdRunner(object): @@ -41,10 +58,6 @@ class _CmdRunner(object): self.commandstr = commandstr def __call__(self, target, source, env): - import SCons.Action - import subprocess - import os - import sys kw = { 'stdin': 'devnull', 'stdout': subprocess.PIPE, @@ -57,11 +70,10 @@ class _CmdRunner(object): self.out, self.err = proc.communicate() self.status = proc.wait() if self.err: - sys.stderr.write(unicode(self.err)) + sys.stderr.write(SCons.Util.UnicodeType(self.err)) return self.status def strfunction(self, target, source, env): - import os comstr = self.commandstr if env.subst(comstr, target=target, source=source) == "": comstr = self.command @@ -74,9 +86,6 @@ class _CmdRunner(object): ############################################################################# def _update_pot_file(target, source, env): """ Action function for `POTUpdate` builder """ - import re - import os - import SCons.Action nop = lambda target, source, env: 0 # Save scons cwd and os cwd (NOTE: they may be different. After the job, we @@ -154,10 +163,6 @@ def _update_pot_file(target, source, env): ############################################################################# ############################################################################# -from SCons.Builder import BuilderBase - - -############################################################################# class _POTBuilder(BuilderBase): def _execute(self, env, target, source, *args): if not target: @@ -175,10 +180,6 @@ class _POTBuilder(BuilderBase): def _scan_xgettext_from_files(target, source, env, files=None, path=None): """ Parses `POTFILES.in`-like file and returns list of extracted file names. """ - import re - import SCons.Util - import SCons.Node.FS - if files is None: return 0 if not SCons.Util.is_List(files): @@ -230,10 +231,6 @@ def _scan_xgettext_from_files(target, source, env, files=None, path=None): ############################################################################# def _pot_update_emitter(target, source, env): """ Emitter function for `POTUpdate` builder """ - from SCons.Tool.GettextCommon import _POTargetFactory - import SCons.Util - import SCons.Node.FS - if 'XGETTEXTFROM' in env: xfrom = env['XGETTEXTFROM'] else: @@ -261,10 +258,6 @@ def _pot_update_emitter(target, source, env): ############################################################################# ############################################################################# -from SCons.Environment import _null - - -############################################################################# def _POTUpdateBuilderWrapper(env, target=None, source=_null, **kw): return env._POTUpdateBuilder(target, source, **kw) @@ -274,8 +267,6 @@ def _POTUpdateBuilderWrapper(env, target=None, source=_null, **kw): ############################################################################# def _POTUpdateBuilder(env, **kw): """ Creates `POTUpdate` builder object """ - import SCons.Action - from SCons.Tool.GettextCommon import _POTargetFactory kw['action'] = SCons.Action.Action(_update_pot_file, None) kw['suffix'] = '$POTSUFFIX' kw['target_factory'] = _POTargetFactory(env, alias='$POTUPDATE_ALIAS').File @@ -288,13 +279,6 @@ def _POTUpdateBuilder(env, **kw): ############################################################################# def generate(env, **kw): """ Generate `xgettext` tool """ - import sys - import os - import SCons.Util - import SCons.Tool - from SCons.Tool.GettextCommon import RPaths, _detect_xgettext - from SCons.Platform.mingw import MINGW_DEFAULT_PATHS - from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS if sys.platform == 'win32': xgettext = SCons.Tool.find_program_path(env, 'xgettext', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) @@ -359,7 +343,6 @@ def generate(env, **kw): ############################################################################# def exists(env): """ Check, whether the tool exists """ - from SCons.Tool.GettextCommon import _xgettext_exists try: return _xgettext_exists(env) except: diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index 07f62ea..baf972f 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -33,6 +33,7 @@ import re import types import codecs import pprint +import hashlib PY3 = sys.version_info[0] == 3 @@ -656,13 +657,15 @@ except ImportError: pass RegError = _NoError -WinError = None + # Make sure we have a definition of WindowsError so we can # run platform-independent tests of Windows functionality on # platforms other than Windows. (WindowsError is, in fact, an # OSError subclass on Windows.) + class PlainWindowsError(OSError): pass + try: WinError = WindowsError except NameError: @@ -1176,10 +1179,13 @@ def unique(s): # ASPN: Python Cookbook: Remove duplicates from a sequence # First comment, dated 2001/10/13. # (Also in the printed Python Cookbook.) +# This not currently used, in favor of the next function... def uniquer(seq, idfun=None): - if idfun is None: - def idfun(x): return x + def default_idfun(x): + return x + if not idfun: + idfun = default_idfun seen = {} result = [] for item in seq: @@ -1444,56 +1450,54 @@ def RenameFunction(function, name): function.__defaults__) -md5 = False +if hasattr(hashlib, 'md5'): + md5 = True + def MD5signature(s): + """ + Generate md5 signature of a string -def MD5signature(s): - return str(s) + :param s: either string or bytes. Normally should be bytes + :return: String of hex digits representing the signature + """ + m = hashlib.md5() + try: + m.update(to_bytes(s)) + except TypeError as e: + m.update(to_bytes(str(s))) -def MD5filesignature(fname, chunksize=65536): - with open(fname, "rb") as f: - result = f.read() - return result + return m.hexdigest() -try: - import hashlib -except ImportError: - pass + def MD5filesignature(fname, chunksize=65536): + """ + Generate the md5 signature of a file + + :param fname: file to hash + :param chunksize: chunk size to read + :return: String of Hex digits representing the signature + """ + m = hashlib.md5() + f = open(fname, "rb") + while True: + blck = f.read(chunksize) + if not blck: + break + m.update(to_bytes(blck)) + f.close() + return m.hexdigest() else: - if hasattr(hashlib, 'md5'): - md5 = True + # if md5 algorithm not available, just return data unmodified + # could add alternative signature scheme here + md5 = False - def MD5signature(s): - """ - Generate a String of Hex digits representing the md5 signature of the string - :param s: either string or bytes. Normally should be bytes - :return: String of hex digits - """ - m = hashlib.md5() + def MD5signature(s): + return str(s) - try: - m.update(to_bytes(s)) - except TypeError as e: - m.update(to_bytes(str(s))) - - return m.hexdigest() - - def MD5filesignature(fname, chunksize=65536): - """ - :param fname: - :param chunksize: - :return: String of Hex digits - """ - m = hashlib.md5() - f = open(fname, "rb") - while True: - blck = f.read(chunksize) - if not blck: - break - m.update(to_bytes(blck)) - f.close() - return m.hexdigest() + def MD5filesignature(fname, chunksize=65536): + with open(fname, "rb") as f: + result = f.read() + return result def MD5collect(signatures): diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index 83b084a..d34243e 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -113,28 +113,28 @@ except AttributeError: # intern into the sys package sys.intern = intern -# Preparing for 3.x. UserDict, UserList, UserString are in -# collections for 3.x, but standalone in 2.7.x +# UserDict, UserList, UserString are in # collections for 3.x, +# but standalone in 2.7.x. Monkey-patch into collections for 2.7. import collections try: collections.UserDict except AttributeError: - exec ('from UserDict import UserDict as _UserDict') + from UserDict import UserDict as _UserDict collections.UserDict = _UserDict del _UserDict try: collections.UserList except AttributeError: - exec ('from UserList import UserList as _UserList') + from UserList import UserList as _UserList collections.UserList = _UserList del _UserList try: collections.UserString except AttributeError: - exec ('from UserString import UserString as _UserString') + from UserString import UserString as _UserString collections.UserString = _UserString del _UserString diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index a413690..c1172aa 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -165,7 +165,7 @@ def CPP_to_Python(s): """ s = CPP_to_Python_Ops_Expression.sub(CPP_to_Python_Ops_Sub, s) for expr, repl in CPP_to_Python_Eval_List: - s = expr.sub(repl, s) + s = re.sub(expr, repl, s) return s -- cgit v0.12 From b774c48825febe7261ef4974f64218c243ca347b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 12:37:38 -0400 Subject: changes to unit test to avoid hitting the filesystem --- src/engine/SCons/Tool/JavaCommon.py | 115 +++++++++++++++++------------- src/engine/SCons/Tool/JavaCommonTests.py | 116 ++++++++++++++++++------------- 2 files changed, 136 insertions(+), 95 deletions(-) diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py index c77ace6..f1c1b4f 100644 --- a/src/engine/SCons/Tool/JavaCommon.py +++ b/src/engine/SCons/Tool/JavaCommon.py @@ -42,6 +42,33 @@ default_java_version = '1.4' # anonymous inner class parsing. scopeStateVersions = ('1.8') +# Glob patterns for use in finding where the JDK is. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +# For now only used for Windows. +java_win32_dir_glob = 'C:/Program Files*/Java/jdk*/bin' +# On windows, since Java 9, there is a dash between 'jdk' and the version +# string that wasn't there before. this glob should catch either way. +java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk*%s*/bin' + +# Glob patterns for use in finding where the JDK headers are. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +java_macos_include_dir_glob = '/System/Library/Frameworks/JavaVM.framework/Headers/' +java_macos_version_include_dir_glob = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' + +java_linux_include_dirs_glob = [ + '/usr/lib/jvm/default-java/include', + '/usr/lib/jvm/java-*/include' +] +# Need to match path like below (from Centos 7) +# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ +java_linux_version_include_dirs_glob = [ + '/usr/lib/jvm/java-*-sun-%s*/include', + '/usr/lib/jvm/java-%s*-openjdk*/include', + '/usr/java/jdk%s*/include' +] + if java_parsing: # Parse Java files for class names. # @@ -64,14 +91,15 @@ if java_parsing: r'\d*\.\d*|[A-Za-z_][\w\$\.]*|<[A-Za-z_]\w+>|' + r'/\*|\*/|\[\])') + class OuterState(object): """The initial state for parsing a Java file for classes, interfaces, and anonymous inner classes.""" + def __init__(self, version=default_java_version): if not version in ('1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '5', '6', '9.0', '10.0', '11.0', '12.0'): - msg = "Java version %s not supported" % version raise NotImplementedError(msg) @@ -131,15 +159,15 @@ if java_parsing: def closeBracket(self): self.brackets = self.brackets - 1 if len(self.stackBrackets) and \ - self.brackets == self.stackBrackets[-1]: + self.brackets == self.stackBrackets[-1]: self.listOutputs.append('$'.join(self.listClasses)) self.localClasses.pop() self.listClasses.pop() self.anonStacksStack.pop() self.stackBrackets.pop() if len(self.stackAnonClassBrackets) and \ - self.brackets == self.stackAnonClassBrackets[-1] and \ - self.version not in scopeStateVersions: + self.brackets == self.stackAnonClassBrackets[-1] and \ + self.version not in scopeStateVersions: self._getAnonStack().pop() self.stackAnonClassBrackets.pop() @@ -152,13 +180,13 @@ if java_parsing: self.openBracket() elif token == '}': self.closeBracket() - elif token in [ '"', "'" ]: + elif token in ['"', "'"]: return IgnoreState(token, self) elif token == "new": # anonymous inner class if len(self.listClasses) > 0: return self.__getAnonClassState() - return self.__getSkipState() # Skip the class name + return self.__getSkipState() # Skip the class name elif token in ['class', 'interface', 'enum']: if len(self.listClasses) == 0: self.nextAnon = 1 @@ -193,11 +221,13 @@ if java_parsing: def setPackage(self, package): self.package = package + class ScopeState(object): """ A state that parses code within a scope normally, within the confines of a scope. """ + def __init__(self, old_state): self.outer_state = old_state.outer_state self.old_state = old_state @@ -259,13 +289,16 @@ if java_parsing: return self.__getSkipState() return self + class AnonClassState(object): """A state that looks for anonymous inner classes.""" + def __init__(self, old_state): # outer_state is always an instance of OuterState self.outer_state = old_state.outer_state self.old_state = old_state self.brace_level = 0 + def parseToken(self, token): # This is an anonymous class if and only if the next # non-whitespace token is a bracket. Everything between @@ -293,26 +326,32 @@ if java_parsing: if token == '{': self.outer_state.addAnonClass() if self.outer_state.version in scopeStateVersions: - return ScopeState(old_state = self.old_state).parseToken(token) + return ScopeState(old_state=self.old_state).parseToken(token) return self.old_state.parseToken(token) + class SkipState(object): """A state that will skip a specified number of tokens before reverting to the previous state.""" + def __init__(self, tokens_to_skip, old_state): self.tokens_to_skip = tokens_to_skip self.old_state = old_state + def parseToken(self, token): self.tokens_to_skip = self.tokens_to_skip - 1 if self.tokens_to_skip < 1: return self.old_state return self + class ClassState(object): """A state we go into when we hit a class or interface keyword.""" + def __init__(self, outer_state): # outer_state is always an instance of OuterState self.outer_state = outer_state + def parseToken(self, token): # the next non-whitespace token should be the name of the class if token == '\n': @@ -322,12 +361,12 @@ if java_parsing: # 'Foo$1Inner' # https://github.com/SCons/scons/issues/2087 if self.outer_state.localClasses and \ - self.outer_state.stackBrackets[-1] > \ - self.outer_state.stackBrackets[-2]+1: + self.outer_state.stackBrackets[-1] > \ + self.outer_state.stackBrackets[-2] + 1: locals = self.outer_state.localClasses[-1] try: idx = locals[token] - locals[token] = locals[token]+1 + locals[token] = locals[token] + 1 except KeyError: locals[token] = 1 token = str(locals[token]) + token @@ -336,32 +375,40 @@ if java_parsing: self.outer_state.anonStacksStack.append([0]) return self.outer_state + class IgnoreState(object): """A state that will ignore all tokens until it gets to a specified token.""" + def __init__(self, ignore_until, old_state): self.ignore_until = ignore_until self.old_state = old_state + def parseToken(self, token): if self.ignore_until == token: return self.old_state return self + class PackageState(object): """The state we enter when we encounter the package keyword. We assume the next token will be the package name.""" + def __init__(self, outer_state): # outer_state is always an instance of OuterState self.outer_state = outer_state + def parseToken(self, token): self.outer_state.setPackage(token) return self.outer_state + def parse_java_file(fn, version=default_java_version): with open(fn, 'r') as f: data = f.read() return parse_java(data, version) + def parse_java(contents, version=default_java_version, trace=None): """Parse a .java file and return a double of package directory, plus a list of .class files that compiling that .java file will @@ -395,15 +442,6 @@ else: return os.path.split(fn) -# Glob patterns for use in finding where the JDK is. -# These are pairs, *dir_glob used in the general case, -# *version_dir_glob if matching only a specific version. -# For now only used for Windows. -java_win32_dir_glob = 'C:/Program Files*/Java/jdk*/bin' -# On windows, since Java 9, there is a dash between 'jdk' and the version -# string that wasn't there before. this glob should catch either way. -java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk*%s*/bin' - def get_java_install_dirs(platform, version=None): """ Find the java jdk installation directories. @@ -419,7 +457,7 @@ def get_java_install_dirs(platform, version=None): paths = [] if platform == 'win32': if version: - paths = glob.glob(java_win32_version_dir_glob%version) + paths = glob.glob(java_win32_version_dir_glob % version) else: paths = glob.glob(java_win32_dir_glob) else: @@ -429,24 +467,6 @@ def get_java_install_dirs(platform, version=None): return sorted(paths) -# Glob patterns for use in finding where the JDK headers are. -# These are pairs, *dir_glob used in the general case, -# *version_dir_glob if matching only a specific version. -java_macos_include_dir = '/System/Library/Frameworks/JavaVM.framework/Headers/' -java_macos_version_include_dir = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' - -java_linux_include_dirs = [ - '/usr/lib/jvm/default-java/include', - '/usr/lib/jvm/java-*/include' -] -# Need to match path like below (from Centos 7) -# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ -java_linux_version_include_dirs = [ - '/usr/lib/jvm/java-*-sun-%s*/include', - '/usr/lib/jvm/java-%s*-openjdk*/include', - '/usr/java/jdk%s*/include' -] - def get_java_include_paths(env, javac, version): """ Find java include paths for JNI building. @@ -467,24 +487,23 @@ def get_java_include_paths(env, javac, version): paths = [java_inc_dir, os.path.join(java_inc_dir, 'win32')] elif env['PLATFORM'] == 'darwin': if not version: - paths = [java_macos_include_dir] + paths = [java_macos_include_dir_glob] else: - paths = sorted(glob.glob(java_macos_version_include_dir%version)) + paths = sorted(glob.glob(java_macos_version_include_dir_glob % version)) else: - base_paths=[] + base_paths = [] if not version: - for p in java_linux_include_dirs: + for p in java_linux_include_dirs_glob: base_paths.extend(glob.glob(p)) else: - for p in java_linux_version_include_dirs: - base_paths.extend(glob.glob(p%version)) + for p in java_linux_version_include_dirs_glob: + base_paths.extend(glob.glob(p % version)) for p in base_paths: - paths.extend([p, os.path.join(p,'linux')]) - - #print("PATHS:%s"%paths) - return paths + paths.extend([p, os.path.join(p, 'linux')]) + # print("PATHS:%s"%paths) + return paths # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py index fe04ce9..9242624 100644 --- a/src/engine/SCons/Tool/JavaCommonTests.py +++ b/src/engine/SCons/Tool/JavaCommonTests.py @@ -26,6 +26,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path import sys import unittest +import fnmatch import SCons.Scanner.IDL import SCons.Tool.JavaCommon @@ -608,54 +609,75 @@ public class AnonDemo { pkg_dir, classes = SCons.Tool.JavaCommon.parse_java(input, '1.8') assert expect == classes, (expect, classes) - def test_jdk_globs(self): - test = TestSCons.TestSCons(workdir='') - - if sys.platform == 'win32': - test.subdir(['Program Files'], - ['Program Files', 'Java'], - ['Program Files', 'Java', 'jdk1.8.0_201'], - ['Program Files', 'Java', 'jdk1.8.0_201', 'bin'], - ['Program Files', 'Java', 'jdk-11.0.2'], - ['Program Files', 'Java', 'jdk-11.0.2', 'bin']) - - test.write(['Program Files', 'Java', 'jdk1.8.0_201', 'bin', 'javac.exe'], "echo Java 1.8") - test.write(['Program Files', 'Java', 'jdk-11.0.2', 'bin', 'javac.exe'], "echo Java 11.0") - else: - test.subdir(['jvm'], - ['jvm', 'java'], - ['jvm', 'java', 'bin'], - ['jvm', 'java-11-openjdk-11.0.2'], - ['jvm', 'java-11-openjdk-11.0.2', 'bin'], - ['jvm', 'java-1.8-openjdk-1.8.0'], - ['jvm', 'java-1.8-openjdk-1.8.0', 'bin']) - - test.write(['jvm', 'java', 'bin', 'javac'], "echo Java 1.8") - test.write(['jvm', 'java-11-openjdk-11.0.2', 'bin', 'javac'], "echo Java 11.0") - test.write(['jvm', 'java-1.8-openjdk-1.8.0', 'bin', 'javac'], "echo Java 1.8") - - for version in (None, "1.8", "11.0"): - if version: - if sys.platform == 'win32': - patterns = [ - 'Program Files*/Java/jdk*%s*/bin' % version, - ] - else: - patterns = [ - 'jvm/*-%s*/bin' % version, - ] - else: - if sys.platform == 'win32': - patterns = [ - 'Program Files*/Java/jdk*/bin', - ] - else: - patterns = [ - 'jvm/*/bin', - ] - java_path = test.paths(patterns) - assert java_path, "no java found by pattern %s" % patterns[0] + """ + Verify that the java path globs work with specific examples. + :return: + """ + from SCons.Tool.JavaCommon import java_linux_include_dirs_glob, java_linux_version_include_dirs_glob, java_win32_dir_glob, java_win32_version_dir_glob, java_macos_include_dir_glob, java_macos_version_include_dir_glob + + # Test windows globs + win_java_dirs = [ + ('C:/Program Files/Java/jdk1.8.0_201/bin', '1.8.0'), + ('C:/Program Files/Java/jdk-11.0.2/bin', '11.0.2'), + ('C:/Program Files/Java/jdk1.7.0_80/bin', '1.7.0') + ] + + for (wjd, version) in win_java_dirs: + if not fnmatch.fnmatch(wjd, java_win32_dir_glob): + self.fail("Didn't properly match %s with pattern %s" % (wjd, java_win32_dir_glob)) + if not fnmatch.fnmatch(wjd, java_win32_version_dir_glob % version): + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, java_win32_version_dir_glob % version)) + + non_win_java_include_dirs = [ + ('/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include', '1.8.0'), + ('/usr/lib/jvm/java-1.8.0-openjdk-amd64/include', '1.8.0'), + ('/usr/lib/jvm/java-8-openjdk-amd64/include', '8'), + ] + + # Test non-windows/non-macos globs + for (wjd, version) in non_win_java_include_dirs: + match = False + globs_tried =[] + for jlig in java_linux_include_dirs_glob: + globs_tried.append(jlig) + + if fnmatch.fnmatch(wjd, jlig): + match = True + break + + if not match: + self.fail("Didn't properly match %s with pattern %s" % (wjd, globs_tried)) + + match = False + globs_tried = [] + for jlvig in java_linux_version_include_dirs_glob: + globs_tried.append(jlvig%version) + if fnmatch.fnmatch(wjd, jlvig % version): + match = True + break + + if not match: + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, globs_tried)) + + # Test macos globs + # Test windows globs + macos_java_dirs = [ + # ('/System/Library/Frameworks/JavaVM.framework/Headers/', None), + ('/System/Library/Frameworks/JavaVM.framework/Versions/11.0.2/Headers/', '11.0.2'), + ] + + if not fnmatch.fnmatch('/System/Library/Frameworks/JavaVM.framework/Headers/', java_macos_include_dir_glob): + self.fail("Didn't properly match %s with pattern %s" % ('/System/Library/Frameworks/JavaVM.framework/Headers/', java_macos_include_dir_glob)) + + for (wjd, version) in macos_java_dirs: + if not fnmatch.fnmatch(wjd, java_macos_version_include_dir_glob % version): + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, java_macos_version_include_dir_glob % version)) + + if __name__ == "__main__": -- cgit v0.12 From 7f6e25d61599ba547008b4edba74cb46b8d43626 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 14:18:09 -0400 Subject: PEP 8 file --- src/engine/SCons/Platform/PlatformTests.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Platform/PlatformTests.py b/src/engine/SCons/Platform/PlatformTests.py index 3f42eae..213c6f7 100644 --- a/src/engine/SCons/Platform/PlatformTests.py +++ b/src/engine/SCons/Platform/PlatformTests.py @@ -33,6 +33,7 @@ import SCons.Platform import SCons.Environment import SCons.Action + class Environment(collections.UserDict): def Detect(self, cmd): return cmd @@ -40,6 +41,7 @@ class Environment(collections.UserDict): def AppendENVPath(self, key, value): pass + class PlatformTestCase(unittest.TestCase): def test_Platform(self): """Test the Platform() function""" @@ -117,6 +119,7 @@ class PlatformTestCase(unittest.TestCase): SCons.Platform.Platform()(env) assert env != {}, env + class TempFileMungeTestCase(unittest.TestCase): def test_MAXLINELENGTH(self): """ Test different values for MAXLINELENGTH with the same @@ -138,18 +141,18 @@ class TempFileMungeTestCase(unittest.TestCase): env['OVERSIMPLIFIED'] = 'command' expanded_cmd = env.subst(defined_cmd) # Call the tempfile munger - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) assert cmd == defined_cmd, cmd # Let MAXLINELENGTH equal the string's length env['MAXLINELENGTH'] = len(expanded_cmd) - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) assert cmd == defined_cmd, cmd # Finally, let the actual tempfile mechanism kick in # Disable printing of actions... old_actions = SCons.Action.print_actions SCons.Action.print_actions = 0 env['MAXLINELENGTH'] = len(expanded_cmd)-1 - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) # ...and restoring its setting. SCons.Action.print_actions = old_actions assert cmd != defined_cmd, cmd @@ -173,9 +176,11 @@ class TempFileMungeTestCase(unittest.TestCase): old_actions = SCons.Action.print_actions SCons.Action.print_actions = 0 # Create an instance of object derived class to allow setattrb - class Node(object) : + + class Node(object): class Attrs(object): pass + def __init__(self): self.attributes = self.Attrs() target = [Node()] @@ -185,6 +190,7 @@ class TempFileMungeTestCase(unittest.TestCase): assert cmd != defined_cmd, cmd assert cmd == getattr(target[0].attributes, 'tempfile_cmdlist', None) + class PlatformEscapeTestCase(unittest.TestCase): def test_posix_escape(self): """ Check that paths with parens are escaped properly -- cgit v0.12 From 2509060d294695b202df6b7a3c12fe0dc42eaecf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 15:15:37 -0400 Subject: Updated mslink, mslib, msvc, platform docs, and added unit test --- src/engine/SCons/Defaults.py | 2 +- src/engine/SCons/Platform/PlatformTests.py | 36 ++++++++++++++++++++++++++++++ src/engine/SCons/Platform/__init__.py | 3 ++- src/engine/SCons/Platform/__init__.xml | 11 +++++++++ src/engine/SCons/Tool/mslib.py | 9 ++++++++ src/engine/SCons/Tool/mslink.py | 5 +++-- src/engine/SCons/Tool/msvc.py | 5 +++-- 7 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index e01b8a0..aa7dd2f 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -582,7 +582,7 @@ ConstructionEnvironment = { '__DSHLIBVERSIONFLAGS' : '${__libversionflags(__env__,"DSHLIBVERSION","_DSHLIBVERSIONFLAGS")}', 'TEMPFILE' : NullCmdGenerator, - 'TEMPFILEARGJOINBYTE': bytearray(' '), + 'TEMPFILEARGJOIN': ' ', 'Dir' : Variable_Method_Caller('TARGET', 'Dir'), 'Dirs' : Variable_Method_Caller('TARGET', 'Dirs'), 'File' : Variable_Method_Caller('TARGET', 'File'), diff --git a/src/engine/SCons/Platform/PlatformTests.py b/src/engine/SCons/Platform/PlatformTests.py index 213c6f7..b906b16 100644 --- a/src/engine/SCons/Platform/PlatformTests.py +++ b/src/engine/SCons/Platform/PlatformTests.py @@ -27,6 +27,7 @@ import SCons.compat import collections import unittest +import os import SCons.Errors import SCons.Platform @@ -157,6 +158,41 @@ class TempFileMungeTestCase(unittest.TestCase): SCons.Action.print_actions = old_actions assert cmd != defined_cmd, cmd + def test_TEMPFILEARGJOINBYTE(self): + """ + Test argument join byte TEMPFILEARGJOINBYTE + """ + + # Init class with cmd, such that the fully expanded + # string reads "a test command line". + # Note, how we're using a command string here that is + # actually longer than the substituted one. This is to ensure + # that the TempFileMunge class internally really takes the + # length of the expanded string into account. + defined_cmd = "a $VERY $OVERSIMPLIFIED line" + t = SCons.Platform.TempFileMunge(defined_cmd) + env = SCons.Environment.SubstitutionEnvironment(tools=[]) + # Setting the line length high enough... + env['MAXLINELENGTH'] = 1024 + env['VERY'] = 'test' + env['OVERSIMPLIFIED'] = 'command' + env['TEMPFILEARGJOINBYTE'] = os.linesep + expanded_cmd = env.subst(defined_cmd) + + # For tempfilemunge to operate. + old_actions = SCons.Action.print_actions + SCons.Action.print_actions = 0 + env['MAXLINELENGTH'] = len(expanded_cmd)-1 + cmd = t(None, None, env, 0) + # print("CMD is:%s"%cmd) + + file_content = open(cmd[-1],'rb').read() + # print("Content is:[%s]"%file_content) + # ...and restoring its setting. + SCons.Action.print_actions = old_actions + assert file_content != env['TEMPFILEARGJOINBYTE'].join(['test','command','line']) + + def test_tempfilecreation_once(self): # Init class with cmd, such that the fully expanded # string reads "a test command line". diff --git a/src/engine/SCons/Platform/__init__.py b/src/engine/SCons/Platform/__init__.py index 068ab89..f864db8 100644 --- a/src/engine/SCons/Platform/__init__.py +++ b/src/engine/SCons/Platform/__init__.py @@ -226,9 +226,10 @@ class TempFileMunge(object): prefix = '@' args = list(map(SCons.Subst.quote_spaces, cmd[1:])) - join_char = env.subst('TEMPFILEARGJOINBYTE') + join_char = env.get('TEMPFILEARGJOIN',bytearray(' ')) os.write(fd, bytearray(join_char.join(args) + "\n",'utf-8')) os.close(fd) + # XXX Using the SCons.Action.print_actions value directly # like this is bogus, but expedient. This class should # really be rewritten as an Action that defines the diff --git a/src/engine/SCons/Platform/__init__.xml b/src/engine/SCons/Platform/__init__.xml index f113278..7ea895e 100644 --- a/src/engine/SCons/Platform/__init__.xml +++ b/src/engine/SCons/Platform/__init__.xml @@ -257,4 +257,15 @@ The default is '.lnk'. + + + +The string (or character) to be used to join the arguments passed to TEMPFILE when command line exceeds the limit set by &cv-MAXLINELENGTH;. +The default value is a space. However for MSVC, MSLINK the default is a line seperator characters as defined by os.linesep. +Note this value is used literally and not expanded by the subst logic. + + + + + diff --git a/src/engine/SCons/Tool/mslib.py b/src/engine/SCons/Tool/mslib.py index c901a75..354f5cf 100644 --- a/src/engine/SCons/Tool/mslib.py +++ b/src/engine/SCons/Tool/mslib.py @@ -33,6 +33,8 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os + import SCons.Defaults import SCons.Tool import SCons.Tool.msvs @@ -54,6 +56,13 @@ def generate(env): env['LIBPREFIX'] = '' env['LIBSUFFIX'] = '.lib' + # Issue #3350 + # Change tempfile argument joining character from a space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOIN'] = os.linesep + + def exists(env): return msvc_exists(env) diff --git a/src/engine/SCons/Tool/mslink.py b/src/engine/SCons/Tool/mslink.py index 176eb79..eae2951 100644 --- a/src/engine/SCons/Tool/mslink.py +++ b/src/engine/SCons/Tool/mslink.py @@ -34,6 +34,7 @@ from __future__ import print_function __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os import os.path import SCons.Action @@ -328,10 +329,10 @@ def generate(env): env['LDMODULECOM'] = compositeLdmodAction # Issue #3350 - # Change tempfile argument joining character from a bytearray space to a newline + # Change tempfile argument joining character from a space to a newline # mslink will fail if any single line is too long, but is fine with many lines # in a tempfile - env['TEMPFILEARGJOINBYTE'] = bytearray('\n') + env['TEMPFILEARGJOIN'] = os.linesep def exists(env): return msvc_exists(env) diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index 91ffa33..dd7d0ec 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -34,6 +34,7 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path +import os import re import sys @@ -284,10 +285,10 @@ def generate(env): msvc_set_PCHPDBFLAGS(env) # Issue #3350 - # Change tempfile argument joining character from a bytearray space to a newline + # Change tempfile argument joining character from a space to a newline # mslink will fail if any single line is too long, but is fine with many lines # in a tempfile - env['TEMPFILEARGJOINBYTE'] = bytearray('\n') + env['TEMPFILEARGJOIN'] = os.linesep env['PCHCOM'] = '$CXX /Fo${TARGETS[1]} $CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS /c $SOURCES /Yc$PCHSTOP /Fp${TARGETS[0]} $CCPDBFLAGS $PCHPDBFLAGS' -- cgit v0.12 From 74a215d49c231069c7b2fd0161d988a63acc8225 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 19:24:02 +0000 Subject: Doc updates --- doc/generated/examples/caching_ex-random_1.xml | 4 ++-- doc/generated/examples/troubleshoot_Dump_1.xml | 1 + doc/generated/examples/troubleshoot_Dump_2.xml | 1 + doc/generated/examples/troubleshoot_explain1_3.xml | 2 +- doc/generated/tools.gen | 12 ++++++------ doc/generated/tools.mod | 4 ++-- doc/generated/variables.gen | 10 ++++++++++ doc/generated/variables.mod | 2 ++ 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index 0a432e9..9f59db5 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,9 +1,9 @@ % scons -Q -cc -o f5.o -c f5.c -cc -o f3.o -c f3.c cc -o f2.o -c f2.c cc -o f1.o -c f1.c +cc -o f5.o -c f5.c cc -o f4.o -c f4.c +cc -o f3.o -c f3.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/examples/troubleshoot_Dump_1.xml b/doc/generated/examples/troubleshoot_Dump_1.xml index 99d1399..99518c0 100644 --- a/doc/generated/examples/troubleshoot_Dump_1.xml +++ b/doc/generated/examples/troubleshoot_Dump_1.xml @@ -57,6 +57,7 @@ scons: Reading SConscript files ... 'TARGET_ARCH': None, 'TARGET_OS': None, 'TEMPFILE': <class 'SCons.Platform.TempFileMunge'>, + 'TEMPFILEARGJOIN': ' ', 'TEMPFILEPREFIX': '@', 'TOOLS': ['install', 'install'], '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__)}', diff --git a/doc/generated/examples/troubleshoot_Dump_2.xml b/doc/generated/examples/troubleshoot_Dump_2.xml index e8e0960..8a7979b 100644 --- a/doc/generated/examples/troubleshoot_Dump_2.xml +++ b/doc/generated/examples/troubleshoot_Dump_2.xml @@ -90,6 +90,7 @@ scons: Reading SConscript files ... 'TARGET_ARCH': None, 'TARGET_OS': None, 'TEMPFILE': <class 'SCons.Platform.TempFileMunge'>, + 'TEMPFILEARGJOIN': '\n', 'TEMPFILEPREFIX': '@', 'TOOLS': ['msvc', 'install', 'install'], '_CCCOMCOM': '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS', diff --git a/doc/generated/examples/troubleshoot_explain1_3.xml b/doc/generated/examples/troubleshoot_explain1_3.xml index ebc13f8..a461f60 100644 --- a/doc/generated/examples/troubleshoot_explain1_3.xml +++ b/doc/generated/examples/troubleshoot_explain1_3.xml @@ -3,5 +3,5 @@ cp file.in file.oout scons: warning: Cannot find target file.out after building -File "/home/bdeegan/devel/scons/git/as_scons/src/script/scons.py", line 204, in <module> +File "/home/bdeegan/devel/scons/git/scons/bootstrap/src/script/scons.py", line 204, in <module> diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index be717e3..ecd9c98 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -779,19 +779,19 @@ Sets construction variables for the Sets: &cv-link-AS;, &cv-link-ASCOM;, &cv-link-ASFLAGS;, &cv-link-ASPPCOM;, &cv-link-ASPPFLAGS;.Uses: &cv-link-ASCOMSTR;, &cv-link-ASPPCOMSTR;. - - packaging + + Packaging -A framework for building binary and source packages. +Sets construction variables for the Package Builder. - - Packaging + + packaging -Sets construction variables for the Package Builder. +A framework for building binary and source packages. diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index f9bc1d7..1209d74 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -78,8 +78,8 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> -packaging"> Packaging"> +packaging"> pdf"> pdflatex"> pdftex"> @@ -186,8 +186,8 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> -packaging"> Packaging"> +packaging"> pdf"> pdflatex"> pdftex"> diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index c7048b4..1c2d2f5 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -7343,6 +7343,16 @@ The suffix used for tar file names. + + TEMPFILEARGJOIN + + +The string (or character) to be used to join the arguments passed to TEMPFILE when command line exceeds the limit set by $MAXLINELENGTH. +The default value is a space. However for MSVC, MSLINK the default is a line seperator characters as defined by os.linesep. +Note this value is used literally and not expanded by the subst logic. + + + TEMPFILEPREFIX diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index 28c08dc..47576f4 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -544,6 +544,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $TARGET_OS"> $TARGETS"> $TARSUFFIX"> +$TEMPFILEARGJOIN"> $TEMPFILEPREFIX"> $TEMPFILESUFFIX"> $TEX"> @@ -1183,6 +1184,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $TARGET_OS"> $TARGETS"> $TARSUFFIX"> +$TEMPFILEARGJOIN"> $TEMPFILEPREFIX"> $TEMPFILESUFFIX"> $TEX"> -- cgit v0.12 From 309db5fc9e6c332d704a9618c01171bfe03815ec Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 15:30:22 -0400 Subject: Fix Issue #3350 - Update CHANGES.txt and RELEASE.txt --- src/CHANGES.txt | 3 +++ src/RELEASE.txt | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 86e98ac..372c2d0 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -10,6 +10,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - Fix Issue #3350 - SCons Exception EnvironmentError is conflicting with Python's EnvironmentError. Renamed to SConsEnvironmentError + - Fix Issue #3350 - mslink failing when too many objects. This is resolved by adding TEMPFILEARGJOIN variable + which specifies what character to join all the argements output into the tempfile. The default remains a space + when mslink, msvc, or mslib tools are loaded they change the TEMPFILEARGJOIN to be a line separator (\r\n on win32) From Mats Wichmann: - scons-time takes more care closing files and uses safer mkdtemp to avoid diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 076e4bb..06051ea 100755 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -31,7 +31,9 @@ NEW FUNCTIONALITY - - List new features (presumably why a checkpoint is being released) + - Added variable TEMPFILEARGJOIN to specify how to join arguments written + to temp files used when command lines exceed MAXLINELENGTH when the + command uses $TEMPFILE{...} DEPRECATED FUNCTIONALITY -- cgit v0.12 From 97f3d7aef76bf73dc33911c7087a2b55df73bd0f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Apr 2019 18:17:00 -0400 Subject: Fix default value logic for TEMPFILEARGJOIN --- src/engine/SCons/Platform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Platform/__init__.py b/src/engine/SCons/Platform/__init__.py index f864db8..7d959b7 100644 --- a/src/engine/SCons/Platform/__init__.py +++ b/src/engine/SCons/Platform/__init__.py @@ -226,7 +226,7 @@ class TempFileMunge(object): prefix = '@' args = list(map(SCons.Subst.quote_spaces, cmd[1:])) - join_char = env.get('TEMPFILEARGJOIN',bytearray(' ')) + join_char = env.get('TEMPFILEARGJOIN',' ') os.write(fd, bytearray(join_char.join(args) + "\n",'utf-8')) os.close(fd) -- cgit v0.12 From 181b480c5223e6656db7768b2be5cc26d7127bc6 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 24 Apr 2019 08:15:35 -0600 Subject: [PR #3337] add testcase for tool lookup Add a unit test to show find_program_path does not alter env['ENV']['PATH']. A little cleanup in Tool/__init__.py: don't use mutable object as default value in function signature (checkers complain about this); getter/setter usage seemed unnecessary - kept one of the two but use modern syntax (checkers complain about use-before set, which is fixed by the change) Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/ToolTests.py | 25 ++++++++++++++++++++-- src/engine/SCons/Tool/__init__.py | 44 ++++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index f005143..f127f91 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -30,11 +30,14 @@ import TestUnit import SCons.Errors import SCons.Tool +from SCons.Environment import Environment + class ToolTestCase(unittest.TestCase): def test_Tool(self): """Test the Tool() function""" - class Environment(object): + + class DummyEnvironment(object): def __init__(self): self.dict = {} def Detect(self, progs): @@ -53,7 +56,8 @@ class ToolTestCase(unittest.TestCase): return key in self.dict def subst(self, string, *args, **kwargs): return string - env = Environment() + + env = DummyEnvironment() env['BUILDERS'] = {} env['ENV'] = {} env['PLATFORM'] = 'test' @@ -78,6 +82,23 @@ class ToolTestCase(unittest.TestCase): raise + def test_pathfind(self): + """Test that find_program_path() does not alter PATH""" + + PHONY_PATHS = [ + r'C:\cygwin64\bin', + r'C:\cygwin\bin', + '/usr/local/dummy/bin', + ] + + # Note this test cannot use the dummy environment, + # as function being tested calls env.WhereIs() + env = Environment() + pre_path = env['ENV']['PATH'] + tool = SCons.Tool.find_program_path(env, 'no_tool', default_paths=PHONY_PATHS) + assert env['ENV']['PATH'] == pre_path, env['ENV']['PATH'] + + if __name__ == "__main__": suite = unittest.makeSuite(ToolTestCase, 'test_') TestUnit.run(suite) diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 166c3b0..cc935d1 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -104,7 +104,9 @@ TOOL_ALIASES = { class Tool(object): - def __init__(self, name, toolpath=[], **kw): + def __init__(self, name, toolpath=None, **kw): + if toolpath is None: + toolpath = [] # Rename if there's a TOOL_ALIAS for this tool self.name = TOOL_ALIASES.get(name, name) @@ -394,7 +396,8 @@ def _call_env_subst(env, string, *args, **kw): class _ShLibInfoSupport(object): - def get_libtype(self): + @property + def libtype(self): return 'ShLib' def get_lib_prefix(self, env, *args, **kw): @@ -411,7 +414,8 @@ class _ShLibInfoSupport(object): class _LdModInfoSupport(object): - def get_libtype(self): + @property + def libtype(self): return 'LdMod' def get_lib_prefix(self, env, *args, **kw): @@ -428,7 +432,8 @@ class _LdModInfoSupport(object): class _ImpLibInfoSupport(object): - def get_libtype(self): + @property + def libtype(self): return 'ImpLib' def get_lib_prefix(self, env, *args, **kw): @@ -480,25 +485,21 @@ class _LibInfoGeneratorBase(object): 'ImpLib': _ImpLibInfoSupport} def __init__(self, libtype, infoname): - self.set_libtype(libtype) - self.set_infoname(infoname) + self.libtype = libtype + self.infoname = infoname + + @property + def libtype(self): + return self._support.libtype - def set_libtype(self, libtype): + @libtype.setter + def libtype(self, libtype): try: support_class = self._support_classes[libtype] except KeyError: raise ValueError('unsupported libtype %r' % libtype) self._support = support_class() - def get_libtype(self): - return self._support.get_libtype() - - def set_infoname(self, infoname): - self.infoname = infoname - - def get_infoname(self): - return self.infoname - def get_lib_prefix(self, env, *args, **kw): return self._support.get_lib_prefix(env, *args, **kw) @@ -518,9 +519,8 @@ class _LibInfoGeneratorBase(object): try: libtype = kw['generator_libtype'] except KeyError: - libtype = self.get_libtype() - infoname = self.get_infoname() - return 'Versioned%s%s' % (libtype, infoname) + libtype = self.libtype + return 'Versioned%s%s' % (libtype, self.infoname) def generate_versioned_lib_info(self, env, args, result=None, **kw): callback = self.get_versioned_lib_info_generator(**kw) @@ -730,7 +730,7 @@ class _LibSonameGenerator(_LibInfoGeneratorBase): if not soname: # fallback to library name (as returned by appropriate _LibNameGenerator) - soname = _LibNameGenerator(self.get_libtype())(env, libnode) + soname = _LibNameGenerator(self.libtype)(env, libnode) if Verbose: print("_LibSonameGenerator: FALLBACK: soname=%r" % soname) @@ -1316,7 +1316,7 @@ def tool_list(platform, env): return [x for x in tools if x] -def find_program_path(env, key_program, default_paths=[]): +def find_program_path(env, key_program, default_paths=None): """ Find the location of a tool using various means. @@ -1338,6 +1338,8 @@ def find_program_path(env, key_program, default_paths=[]): # Finally, add the defaults and check again. Do not change # ['ENV']['PATH'] permananetly, the caller can do that if needed. + if default_paths is None: + return path save_path = env['ENV']['PATH'] for p in default_paths: env.AppendENVPath('PATH', p) -- cgit v0.12 From f270a531be34c4841646e1b1b6c879f47986d5b5 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 24 Apr 2019 09:37:59 -0600 Subject: [PR #3337] test case now uses mocked env Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/ToolTests.py | 66 ++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index f127f91..6cc1724 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -23,6 +23,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os import sys import unittest @@ -33,30 +34,47 @@ import SCons.Tool from SCons.Environment import Environment +class DummyEnvironment(object): + def __init__(self): + self.dict = {} + def Detect(self, progs): + if not SCons.Util.is_List(progs): + progs = [ progs ] + return progs[0] + def Append(self, **kw): + self.dict.update(kw) + def __getitem__(self, key): + return self.dict[key] + def __setitem__(self, key, val): + self.dict[key] = val + def __contains__(self, key): + return self.dict.__contains__(key) + def has_key(self, key): + return key in self.dict + def subst(self, string, *args, **kwargs): + return string + + PHONY_PATH = "/usr/phony/bin" + def WhereIs(self, key_program): + # for pathfind test for Issue #3336: + # need to fake the case where extra paths are searched, and + # if one has a "hit" after some fails, the fails are left in + # the environment's PATH. So construct a positive answer if + # we see a magic known path component in PATH; answer in + # the negative otherwise. + paths = self['ENV']['PATH'] + if self.PHONY_PATH in paths: + return os.path.join(self.PHONY_PATH, key_program) + return None + def AppendENVPath(self, pathvar, path): + # signature matches how called from find_program_path() + self['ENV'][pathvar] = self['ENV'][pathvar] + os.pathsep + path + + class ToolTestCase(unittest.TestCase): def test_Tool(self): """Test the Tool() function""" - class DummyEnvironment(object): - def __init__(self): - self.dict = {} - def Detect(self, progs): - if not SCons.Util.is_List(progs): - progs = [ progs ] - return progs[0] - def Append(self, **kw): - self.dict.update(kw) - def __getitem__(self, key): - return self.dict[key] - def __setitem__(self, key, val): - self.dict[key] = val - def __contains__(self, key): - return self.dict.__contains__(key) - def has_key(self, key): - return key in self.dict - def subst(self, string, *args, **kwargs): - return string - env = DummyEnvironment() env['BUILDERS'] = {} env['ENV'] = {} @@ -85,15 +103,15 @@ class ToolTestCase(unittest.TestCase): def test_pathfind(self): """Test that find_program_path() does not alter PATH""" + env = DummyEnvironment() PHONY_PATHS = [ r'C:\cygwin64\bin', r'C:\cygwin\bin', '/usr/local/dummy/bin', + env.PHONY_PATH, # will be recognized by dummy WhereIs ] - - # Note this test cannot use the dummy environment, - # as function being tested calls env.WhereIs() - env = Environment() + env['ENV'] = {} + env['ENV']['PATH'] = '/usr/local/bin:/opt/bin:/bin:/usr/bin' pre_path = env['ENV']['PATH'] tool = SCons.Tool.find_program_path(env, 'no_tool', default_paths=PHONY_PATHS) assert env['ENV']['PATH'] == pre_path, env['ENV']['PATH'] -- cgit v0.12 From 511e9966835a3b13b1575092ae5190ef001638c9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 24 Apr 2019 13:56:01 -0600 Subject: [PR #3337] fix two minor Sider complaints Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/ToolTests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index 6cc1724..74524f1 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -31,7 +31,6 @@ import TestUnit import SCons.Errors import SCons.Tool -from SCons.Environment import Environment class DummyEnvironment(object): @@ -113,7 +112,7 @@ class ToolTestCase(unittest.TestCase): env['ENV'] = {} env['ENV']['PATH'] = '/usr/local/bin:/opt/bin:/bin:/usr/bin' pre_path = env['ENV']['PATH'] - tool = SCons.Tool.find_program_path(env, 'no_tool', default_paths=PHONY_PATHS) + _ = SCons.Tool.find_program_path(env, 'no_tool', default_paths=PHONY_PATHS) assert env['ENV']['PATH'] == pre_path, env['ENV']['PATH'] -- cgit v0.12