Merge branch 'master' into runtest-devmode

16 files changed, 645 insertions, 365 deletions

diff --git a/runtest.py b/runtest.py
index 6e4f58d..3edf36c 100755
--- a/runtest.py
+++ b/runtest.py
@@ -67,7 +67,7 @@ Options:
 
 Environment Variables:
   PRESERVE, PRESERVE_{PASS,FAIL,NO_RESULT}: preserve test subdirs
-  TESTCMD_VERBOSE: turn on verbosity in TestCommand\
+  TESTCMD_VERBOSE: turn on verbosity in TestCommand
 """
 
 import getopt
@@ -75,37 +75,36 @@ import glob
 import os
 import re
 import stat
+import subprocess
 import sys
-import time
-
+import tempfile
 import threading
-try:                        # python3
-    from queue import Queue
-except ImportError as e:    # python2
-    from Queue import Queue
-import subprocess
+import time
+from abc import ABC, abstractmethod
+from optparse import OptionParser, BadOptionError
+from queue import Queue
 
 cwd = os.getcwd()
 
-baseline = 0
+baseline = None
 builddir = os.path.join(cwd, 'build')
 external = 0
 devmode = False
 debug = ''
-execute_tests = 1
+execute_tests = True
 jobs = 1
-list_only = None
-printcommand = 1
+list_only = False
+printcommand = True
 package = None
-print_passed_summary = None
+print_passed_summary = False
 scons = None
-scons_exec = None
+scons_exec = False
 testlistfile = None
 version = ''
-print_times = None
+print_times = False
 python = None
 sp = []
-print_progress = 1
+print_progress = True
 catch_output = False
 suppress_output = False
 allow_pipe_files = True
@@ -124,8 +123,6 @@ helpstr = usagestr + __doc__
 # unknown options and lets them pile up in the leftover argument
 # list.  Useful to gradually port getopt to optparse.
 
-from optparse import OptionParser, BadOptionError
-
 class PassThroughOptionParser(OptionParser):
     def _process_long_opt(self, rargs, values):
         try:
@@ -139,32 +136,47 @@ class PassThroughOptionParser(OptionParser):
             self.largs.append(err.opt_str)
 
 parser = PassThroughOptionParser(add_help_option=False)
-parser.add_option('-a', '--all', action='store_true',
-                      help="Run all tests.")
+parser.add_option('-a', '--all', action='store_true', help="Run all tests.")
 parser.add_option('-o', '--output',
-                      help="Save the output from a test run to the log file.")
+                  help="Save the output from a test run to the log file.")
 parser.add_option('--runner', metavar='class',
-                      help="Test runner class for unit tests.")
-parser.add_option('--xml',
-                      help="Save results to file in SCons XML format.")
+                  help="Test runner class for unit tests.")
+parser.add_option('--xml', help="Save results to file in SCons XML format.")
 (options, args) = parser.parse_args()
 
-#print("options:", options)
-#print("args:", args)
-
-
-opts, args = getopt.getopt(args, "b:dDef:hj:klnP:p:qsv:Xx:t",
-                            ['baseline=', 'builddir=',
-                             'debug', 'devmode', 'external',
-                             'file=', 'help', 'no-progress',
-                             'jobs=',
-                             'list', 'no-exec', 'nopipefiles',
-                             'package=', 'passed', 'python=',
-                             'quiet',
-                             'quit-on-failure',
-                             'short-progress', 'time',
-                             'version=', 'exec=',
-                             'verbose=', 'exclude-list='])
+# print("options:", options)
+# print("args:", args)
+
+
+opts, args = getopt.getopt(
+    args,
+    "b:dDef:hj:klnP:p:qsv:Xx:t",
+    [
+        "baseline=",
+        "builddir=",
+        "debug",
+        "devmode", 
+        "external",
+        "file=",
+        "help",
+        "no-progress",
+        "jobs=",
+        "list",
+        "no-exec",
+        "nopipefiles",
+        "package=",
+        "passed",
+        "python=",
+        "quiet",
+        "quit-on-failure",
+        "short-progress",
+        "time",
+        "version=",
+        "exec=",
+        "verbose=",
+        "exclude-list=",
+    ],
+)
 
 for o, a in opts:
     if o in ['-b', '--baseline']:
@@ -174,15 +186,15 @@ for o, a in opts:
         if not os.path.isabs(builddir):
             builddir = os.path.normpath(os.path.join(cwd, builddir))
     elif o in ['-d', '--debug']:
-        for dir in sys.path:
-            pdb = os.path.join(dir, 'pdb.py')
+        for d in sys.path:
+            pdb = os.path.join(d, 'pdb.py')
             if os.path.exists(pdb):
                 debug = pdb
                 break
     elif o in ['-D', '--devmode']:
         devmode = True
     elif o in ['-e', '--external']:
-        external = 1
+        external = True
     elif o in ['-f', '--file']:
         if not os.path.isabs(a):
             a = os.path.join(cwd, a)
@@ -196,46 +208,45 @@ for o, a in opts:
         # or outputs will interleave and be hard to read
         catch_output = True
     elif o in ['-k', '--no-progress']:
-        print_progress = 0
+        print_progress = False
     elif o in ['-l', '--list']:
-        list_only = 1
+        list_only = True
     elif o in ['-n', '--no-exec']:
-        execute_tests = None
+        execute_tests = False
     elif o in ['--nopipefiles']:
         allow_pipe_files = False
     elif o in ['-p', '--package']:
         package = a
     elif o in ['--passed']:
-        print_passed_summary = 1
+        print_passed_summary = True
     elif o in ['-P', '--python']:
         python = a
     elif o in ['-q', '--quiet']:
-        printcommand = 0
+        printcommand = False
         suppress_output = catch_output = True
     elif o in ['--quit-on-failure']:
         quit_on_failure = True
     elif o in ['-s', '--short-progress']:
-        print_progress = 1
+        print_progress = True
         suppress_output = catch_output = True
     elif o in ['-t', '--time']:
-        print_times = 1
+        print_times = True
     elif o in ['--verbose']:
         os.environ['TESTCMD_VERBOSE'] = a
     elif o in ['-v', '--version']:
         version = a
     elif o in ['-X']:
-        scons_exec = 1
+        scons_exec = True
     elif o in ['-x', '--exec']:
         scons = a
     elif o in ['--exclude-list']:
         excludelistfile = a
 
 
-# --- setup stdout/stderr ---
-class Unbuffered(object):
+class Unbuffered():
+    """ class to arrange for stdout/stderr to be unbuffered """
     def __init__(self, file):
         self.file = file
-        self.softspace = 0  ## backward compatibility; not supported in Py3k
     def write(self, arg):
         self.file.write(arg)
         self.file.flush()
@@ -245,9 +256,12 @@ class Unbuffered(object):
 sys.stdout = Unbuffered(sys.stdout)
 sys.stderr = Unbuffered(sys.stderr)
 
+# possible alternative: switch to using print, and:
+# print = functools.partial(print, flush)
+
 if options.output:
     logfile = open(options.output, 'w')
-    class Tee(object):
+    class Tee():
         def __init__(self, openfile, stream):
             self.file = openfile
             self.stream = stream
@@ -259,11 +273,10 @@ if options.output:
 
 # --- define helpers ----
 if sys.platform in ('win32', 'cygwin'):
-
     def whereis(file):
         pathext = [''] + os.environ['PATHEXT'].split(os.pathsep)
-        for dir in os.environ['PATH'].split(os.pathsep):
-            f = os.path.join(dir, file)
+        for d in os.environ['PATH'].split(os.pathsep):
+            f = os.path.join(d, file)
             for ext in pathext:
                 fext = f + ext
                 if os.path.isfile(fext):
@@ -271,10 +284,9 @@ if sys.platform in ('win32', 'cygwin'):
         return None
 
 else:
-
     def whereis(file):
-        for dir in os.environ['PATH'].split(os.pathsep):
-            f = os.path.join(dir, file)
+        for d in os.environ['PATH'].split(os.pathsep):
+            f = os.path.join(d, file)
             if os.path.isfile(f):
                 try:
                     st = os.stat(f)
@@ -302,8 +314,8 @@ if not catch_output:
     # Without any output suppressed, we let the subprocess
     # write its stuff freely to stdout/stderr.
     def spawn_it(command_args):
-        p = subprocess.Popen(command_args, shell=False)
-        return (None, None, p.wait())
+        cp = subprocess.run(command_args, shell=False)
+        return cp.stdout, cp.stderr, cp.returncode
 else:
     # Else, we catch the output of both pipes...
     if allow_pipe_files:
@@ -320,16 +332,13 @@ else:
         # subprocess.PIPE.
         def spawn_it(command_args):
             # Create temporary files
-            import tempfile
             tmp_stdout = tempfile.TemporaryFile(mode='w+t')
             tmp_stderr = tempfile.TemporaryFile(mode='w+t')
             # Start subprocess...
-            p = subprocess.Popen(command_args,
-                                 stdout=tmp_stdout,
-                                 stderr=tmp_stderr,
-                                 shell=False)
-            # ... and wait for it to finish.
-            ret = p.wait()
+            cp = subprocess.run(command_args,
+                                stdout=tmp_stdout,
+                                stderr=tmp_stderr,
+                                shell=False)
 
             try:
                 # Rewind to start of files
@@ -344,7 +353,7 @@ else:
                 tmp_stderr.close()
 
             # Return values
-            return (spawned_stderr, spawned_stdout, ret)
+            return spawned_stderr, spawned_stdout, cp.returncode
 
     else:
         # We get here only if the user gave the '--nopipefiles'
@@ -360,30 +369,37 @@ else:
         #   Hence a deadlock.
         # Be dragons here! Better don't use this!
         def spawn_it(command_args):
-            p = subprocess.Popen(command_args,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE,
-                                 shell=False)
-            spawned_stdout = p.stdout.read()
-            spawned_stderr = p.stderr.read()
-            return (spawned_stderr, spawned_stdout, p.wait())
+            cp = subprocess.run(command_args,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                shell=False)
+            return cp.stdout, cp.stderr, cp.returncode
 
 
-class RuntestBase(object):
+class RuntestBase(ABC):
+    """ Base class for tests """
     def __init__(self, path, num, spe=None):
         self.path = path
         self.num = num
+        self.stdout = self.stderr = self.status = None
         self.abspath = os.path.abspath(path)
+        self.command_args = []
+        self.command_str = ""
+        self.test_time = self.total_time = 0
         if spe:
-            for dir in spe:
-                f = os.path.join(dir, path)
+            for d in spe:
+                f = os.path.join(d, path)
                 if os.path.isfile(f):
                     self.abspath = f
                     break
-        self.status = None
+
+    @abstractmethod
+    def execute(self):
+        pass
 
 
 class SystemExecutor(RuntestBase):
+    """ Test class for tests executed with spawn_it() """
     def execute(self):
         self.stderr, self.stdout, s = spawn_it(self.command_args)
         self.status = s
@@ -392,22 +408,28 @@ class SystemExecutor(RuntestBase):
 
 
 class PopenExecutor(RuntestBase):
+    """ Test class for tests executed with Popen
+
+    A bit of a misnomer as the Popen call is now wrapped
+    by calling subprocess.run (behind the covers uses Popen.
+    Very similar to SystemExecutor, but uses command_str
+    instead of command_args, and doesn't allow for not catching
+    the output.
+    """
     # For an explanation of the following 'if ... else'
     # and the 'allow_pipe_files' option, please check out the
     # definition of spawn_it() above.
     if allow_pipe_files:
         def execute(self):
             # Create temporary files
-            import tempfile
             tmp_stdout = tempfile.TemporaryFile(mode='w+t')
             tmp_stderr = tempfile.TemporaryFile(mode='w+t')
             # Start subprocess...
-            p = subprocess.Popen(self.command_str.split(),
-                                 stdout=tmp_stdout,
-                                 stderr=tmp_stderr,
-                                 shell=False)
-            # ... and wait for it to finish.
-            self.status = p.wait()
+            cp = subprocess.run(self.command_str.split(),
+                                stdout=tmp_stdout,
+                                stderr=tmp_stderr,
+                                shell=False)
+            self.status = cp.returncode
 
             try:
                 # Rewind to start of files
@@ -422,19 +444,20 @@ class PopenExecutor(RuntestBase):
                 tmp_stderr.close()
     else:
         def execute(self):
-            p = subprocess.Popen(self.command_str.split(),
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE,
-                                 shell=False)
-            self.status = p.wait()
-            with p.stdout:
-                self.stdout = p.stdout.read()
-            with p.stderr:
-                self.stderr = p.stderr.read()
+            cp = subprocess.run(self.command_str.split(),
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                shell=False)
+            self.status = cp.returncode
+            self.stdout = cp.stdout
+            self.stderr = cp.stderr
 
 class XML(PopenExecutor):
-    def header(self, f):
+    """ Test class for tests that will output in scons xml """
+    @staticmethod
+    def header(f):
         f.write('  <results>\n')
+
     def write(self, f):
         f.write('    <test>\n')
         f.write('      <file_name>%s</file_name>\n' % self.path)
@@ -444,6 +467,7 @@ class XML(PopenExecutor):
         f.write('      <stderr>%s</stderr>\n' % self.stderr)
         f.write('      <time>%.1f</time>\n' % self.test_time)
         f.write('    </test>\n')
+
     def footer(self, f):
         f.write('  <time>%.1f</time>\n' % self.total_time)
         f.write('  </results>\n')
@@ -456,7 +480,7 @@ else:
 # --- start processing ---
 if package:
 
-    dir = {
+    dirs = {
         'deb'          : 'usr',
         'local-tar-gz' : None,
         'local-zip'    : None,
@@ -473,13 +497,13 @@ if package:
         'deb'        : os.path.join('python2.1', 'site-packages')
     }
 
-    if package not in dir:
+    if package not in dirs:
         sys.stderr.write("Unknown package '%s'\n" % package)
         sys.exit(2)
 
     test_dir = os.path.join(builddir, 'test-%s' % package)
 
-    if dir[package] is None:
+    if dirs[package] is None:
         scons_script_dir = test_dir
         globs = glob.glob(os.path.join(test_dir, 'scons-local-*'))
         if not globs:
@@ -488,13 +512,13 @@ if package:
         scons_lib_dir = None
         pythonpath_dir = globs[len(globs)-1]
     elif sys.platform == 'win32':
-        scons_script_dir = os.path.join(test_dir, dir[package], 'Scripts')
-        scons_lib_dir = os.path.join(test_dir, dir[package])
+        scons_script_dir = os.path.join(test_dir, dirs[package], 'Scripts')
+        scons_lib_dir = os.path.join(test_dir, dirs[package])
         pythonpath_dir = scons_lib_dir
     else:
-        scons_script_dir = os.path.join(test_dir, dir[package], 'bin')
-        l = lib.get(package, 'scons')
-        scons_lib_dir = os.path.join(test_dir, dir[package], 'lib', l)
+        scons_script_dir = os.path.join(test_dir, dirs[package], 'bin')
+        sconslib = lib.get(package, 'scons')
+        scons_lib_dir = os.path.join(test_dir, dirs[package], 'lib', sconslib)
         pythonpath_dir = scons_lib_dir
 
     scons_runtest_dir = builddir
@@ -508,7 +532,7 @@ else:
     elif baseline == '-':
         url = None
         with os.popen("svn info 2>&1", "r") as p:
-            svn_info =  p.read()
+            svn_info = p.read()
         match = re.search(r'URL: (.*)', svn_info)
         if match:
             url = match.group(1)
@@ -516,7 +540,6 @@ else:
             sys.stderr.write('runtest.py: could not find a URL:\n')
             sys.stderr.write(svn_info)
             sys.exit(1)
-        import tempfile
         base = tempfile.mkdtemp(prefix='runtest-tmp-')
 
         command = 'cd %s && svn co -q %s' % (base, url)
@@ -571,7 +594,7 @@ if '_JAVA_OPTIONS' in os.environ:
 #        harness from $srcdir/etc. Those modules should be transfered
 #        to testing/, in which case this manipulation of PYTHONPATH
 #        should be able to go away.
-pythonpaths = [ pythonpath_dir ]
+pythonpaths = [pythonpath_dir]
 
 scriptpath = os.path.dirname(os.path.realpath(__file__))
 
@@ -595,7 +618,7 @@ unittests = []
 endtests = []
 
 
-def find_Tests_py(directory):
+def find_unit_tests(directory):
     """ Look for unit tests """
     result = []
     for dirpath, dirnames, filenames in os.walk(directory):
@@ -608,7 +631,7 @@ def find_Tests_py(directory):
     return sorted(result)
 
 
-def find_py(directory):
+def find_e2e_tests(directory):
     """ Look for end-to-end tests """
     result = []
 
@@ -633,8 +656,7 @@ if testlistfile:
     tests = [x for x in tests if x[0] != '#']
     tests = [x[:-1] for x in tests]
     tests = [x.strip() for x in tests]
-    tests = [x for x in tests if len(x) > 0]
-
+    tests = [x for x in tests if x]
 else:
     testpaths = []
 
@@ -666,10 +688,10 @@ else:
         for path in glob.glob(tp):
             if os.path.isdir(path):
                 if path.startswith('src') or path.startswith('testing'):
-                    for p in find_Tests_py(path):
+                    for p in find_unit_tests(path):
                         unittests.append(p)
                 elif path.startswith('test'):
-                    for p in find_py(path):
+                    for p in find_e2e_tests(path):
                         endtests.append(p)
             else:
                 if path.endswith("Tests.py"):
@@ -695,11 +717,11 @@ if excludelistfile:
     excludetests = [x for x in excludetests if x[0] != '#']
     excludetests = [x[:-1] for x in excludetests]
     excludetests = [x.strip() for x in excludetests]
-    excludetests = [x for x in excludetests if len(x) > 0]
+    excludetests = [x for x in excludetests if x]
 
 # ---[ test processing ]-----------------------------------
 tests = [t for t in tests if t not in excludetests]
-tests = [Test(t, n+1) for n, t in enumerate(tests)]
+tests = [Test(t, n + 1) for n, t in enumerate(tests)]
 
 if list_only:
     for t in tests:
@@ -713,24 +735,14 @@ if not python:
         python = sys.executable
 os.environ["python_executable"] = python
 
-# time.clock() is the suggested interface for doing benchmarking timings,
-# but time.time() does a better job on Linux systems, so let that be
-# the non-Windows default.
-
-#TODO: clean up when py2 support is dropped
-try:
-    time_func = time.perf_counter
-except AttributeError:
-    if sys.platform == 'win32':
-        time_func = time.clock
-    else:
-        time_func = time.time
-
 if print_times:
-    print_time_func = lambda fmt, time: sys.stdout.write(fmt % time)
+    def print_time(fmt, tm):
+        sys.stdout.write(fmt % tm)
 else:
-    print_time_func = lambda fmt, time: None
+    def print_time(fmt, tm):
+        pass
 
+time_func = time.perf_counter
 total_start_time = time_func()
 total_num_tests = len(tests)
 
@@ -761,7 +773,7 @@ def log_result(t, io_lock=None):
                 print(t.stdout)
             if t.stderr:
                 print(t.stderr)
-        print_time_func("Test execution time: %.1f seconds\n", t.test_time)
+        print_time("Test execution time: %.1f seconds\n", t.test_time)
     finally:
         if io_lock:
             io_lock.release()
@@ -787,10 +799,13 @@ def run_test(t, io_lock=None, run_async=True):
     t.command_str = " ".join([escape(python)] + command_args)
     if printcommand:
         if print_progress:
-            t.headline += ("%d/%d (%.2f%s) %s\n" % (t.num, total_num_tests,
-                                                float(t.num)*100.0/float(total_num_tests),
-                                                '%',
-                                                t.command_str))
+            t.headline += "%d/%d (%.2f%s) %s\n" % (
+                t.num,
+                total_num_tests,
+                float(t.num) * 100.0 / float(total_num_tests),
+                "%",
+                t.command_str,
+            )
         else:
             t.headline += t.command_str + "\n"
     if not suppress_output and not catch_output:
@@ -812,6 +827,10 @@ def run_test(t, io_lock=None, run_async=True):
 
 
 class RunTest(threading.Thread):
+    """ Test Runner class
+
+    One instance will be created for each job thread in multi-job mode
+    """
     def __init__(self, queue=None, io_lock=None,
                  group=None, target=None, name=None, args=(), kwargs=None):
         super(RunTest, self).__init__(group=group, target=target, name=name)
@@ -825,25 +844,25 @@ class RunTest(threading.Thread):
 
 if jobs > 1:
     print("Running tests using %d jobs" % jobs)
-    queue = Queue()
+    testq = Queue()
     for t in tests:
-        queue.put(t)
-    io_lock = threading.Lock()
+        testq.put(t)
+    testlock = threading.Lock()
     # Start worker threads to consume the queue
-    threads = [RunTest(queue=queue, io_lock=io_lock) for _ in range(jobs)]
+    threads = [RunTest(queue=testq, io_lock=testlock) for _ in range(jobs)]
     for t in threads:
         t.daemon = True
         t.start()
     # wait on the queue rather than the individual threads
-    queue.join()
+    testq.join()
 else:
     for t in tests:
         run_test(t, io_lock=None, run_async=False)
 
 # --- all tests are complete by the time we get here ---
-if len(tests) > 0:
+if tests:
     tests[0].total_time = time_func() - total_start_time
-    print_time_func("Total execution time for all tests: %.1f seconds\n", tests[0].total_time)
+    print_time("Total execution time for all tests: %.1f seconds\n", tests[0].total_time)
 
 passed = [t for t in tests if t.status == 0]
 fail = [t for t in tests if t.status == 1]
@@ -892,9 +911,9 @@ if options.output:
     if isinstance(sys.stderr, Tee):
         sys.stderr.file.close()
 
-if len(fail):
+if fail:
     sys.exit(1)
-elif len(no_result):
+elif no_result:
     sys.exit(2)
 else:
     sys.exit(0)
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 9d2f51c..7c5c9c2 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -10,6 +10,10 @@ NOTE: Please include a reference to any Issues resolved by your changes in the b
 
 RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
+From Rob Boehne
+    - Specify UTF-8 encoding when opening Java source file as text.  By default, encoding is the output
+    of locale.getpreferredencoding(False), and varies by platform.
+
   From William Deegan:
     - Fix broken clang + MSVC 2019 combination by using MSVC configuration logic to 
       propagate'VCINSTALLDIR' and 'VCToolsInstallDir' which clang tools use to locate
@@ -28,6 +32,11 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
     - Updated design doc to use the correct syntax for Depends()
 
   From Adam Gross:
+    - Added support for scanning multiple entries in an action string if
+      IMPLICIT_COMMAND_DEPENDENCIES is set to 2 or 'all'. This enables more thorough
+      action scanning where every item in each command line is scanned to determine
+      if it is a non-source and non-target path and added to the list of implicit dependencies
+      for the target.
     - Added support for taking instances of the Value class as implicit
       dependencies.
     - Added new module SCons.Scanner.Python to allow scanning .py files.
@@ -70,6 +79,7 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
     - Add an alternate warning message cl.exe is not found and msvc config
       cache is in use (SCONS_CACHE_MSVC_CONFIG was given) - config cache
       may be out of date.
+    - Fixed bug where changing TEXTFILESUFFIX would cause Substfile() to rebuild. (Github Issue #3540)
     - Script/Main.py now uses importlib instead of imp module.
 
 
diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py
index 0b7282c..8a8cf27 100644
--- a/src/engine/SCons/Action.py
+++ b/src/engine/SCons/Action.py
@@ -968,11 +968,33 @@ class CommandAction(_ActionAction):
             return env.subst_target_source(cmd, SUBST_SIG, target, source)
 
     def get_implicit_deps(self, target, source, env, executor=None):
+        """Return the implicit dependencies of this action's command line."""
         icd = env.get('IMPLICIT_COMMAND_DEPENDENCIES', True)
         if is_String(icd) and icd[:1] == '$':
             icd = env.subst(icd)
+
         if not icd or icd in ('0', 'None'):
             return []
+
+        try:
+            icd_int = int(icd)
+        except ValueError:
+            icd_int = None
+
+        if (icd_int and icd_int > 1) or icd == 'all':
+            # An integer value greater than 1 specifies the number of entries
+            # to scan. "all" means to scan all.
+            return self._get_implicit_deps_heavyweight(target, source, env, executor, icd_int)
+        else:
+            # Everything else (usually 1 or True) means that we want
+            # lightweight dependency scanning.
+            return self._get_implicit_deps_lightweight(target, source, env, executor)
+
+    def _get_implicit_deps_lightweight(self, target, source, env, executor):
+        """
+        Lightweight dependency scanning involves only scanning the first entry
+        in an action string, even if it contains &&.
+        """
         from SCons.Subst import SUBST_SIG
         if executor:
             cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, executor=executor)
@@ -990,6 +1012,65 @@ class CommandAction(_ActionAction):
                     res.append(env.fs.File(d))
         return res
 
+    def _get_implicit_deps_heavyweight(self, target, source, env, executor,
+                                       icd_int):
+        """
+        Heavyweight dependency scanning involves scanning more than just the
+        first entry in an action string. The exact behavior depends on the
+        value of icd_int. Only files are taken as implicit dependencies;
+        directories are ignored.
+
+        If icd_int is an integer value, it specifies the number of entries to
+        scan for implicit dependencies. Action strings are also scanned after
+        a &&. So for example, if icd_int=2 and the action string is
+        "cd <some_dir> && $PYTHON $SCRIPT_PATH <another_path>", the implicit
+        dependencies would be the path to the python binary and the path to the
+        script.
+
+        If icd_int is None, all entries are scanned for implicit dependencies.
+        """
+
+        # Avoid circular and duplicate dependencies by not providing source,
+        # target, or executor to subst_list. This causes references to
+        # $SOURCES, $TARGETS, and all related variables to disappear.
+        from SCons.Subst import SUBST_SIG
+        cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, conv=lambda x: x)
+        res = []
+
+        for cmd_line in cmd_list:
+            if cmd_line:
+                entry_count = 0
+                for entry in cmd_line:
+                    d = str(entry)
+                    if ((icd_int is None or entry_count < icd_int) and
+                            not d.startswith(('&', '-', '/') if os.name == 'nt'
+                                             else ('&', '-'))):
+                        m = strip_quotes.match(d)
+                        if m:
+                            d = m.group(1)
+
+                        if d:
+                            # Resolve the first entry in the command string using
+                            # PATH, which env.WhereIs() looks in.
+                            # For now, only match files, not directories.
+                            p = os.path.abspath(d) if os.path.isfile(d) else None
+                            if not p and entry_count == 0:
+                                p = env.WhereIs(d)
+
+                            if p:
+                                res.append(env.fs.File(p))
+
+                        entry_count = entry_count + 1
+                    else:
+                        entry_count = 0 if d == '&&' else entry_count + 1
+
+        # Despite not providing source and target to env.subst() above, we
+        # can still end up with sources in this list. For example, files in
+        # LIBS will still resolve in env.subst(). This won't result in
+        # circular dependencies, but it causes problems with cache signatures
+        # changing between full and incremental builds.
+        return [r for r in res if r not in target and r not in source]
+
 
 class CommandGeneratorAction(ActionBase):
     """Class for command-generator actions."""
diff --git a/src/engine/SCons/Action.xml b/src/engine/SCons/Action.xml
index 7a8194e..d85af3b 100644
--- a/src/engine/SCons/Action.xml
+++ b/src/engine/SCons/Action.xml
@@ -58,6 +58,45 @@ not be added to the targets
 built with that construction environment.
 </para>
 
+<para>
+If the construction variable
+&cv-IMPLICIT_COMMAND_DEPENDENCIES;
+is set to <literal>2</literal> or higher,
+then that number of entries in the command
+string will be scanned for relative or absolute
+paths. The count will reset after any
+<literal>&&</literal> entries are found.
+The first command in the action string and
+the first after any <literal>&&</literal>
+entries will be found by searching the
+<varname>PATH</varname> variable in the
+<varname>ENV</varname> environment used to
+execute the command. All other commands will
+only be found if they are absolute paths or
+valid paths relative to the working directory.
+</para>
+
+<para>
+If the construction variable
+&cv-IMPLICIT_COMMAND_DEPENDENCIES;
+is set to <literal>all</literal>, then
+all entries in all command strings will be
+scanned for relative or absolute paths. If
+any are present, they will be added as
+implicit dependencies to the targets built
+with that construction environment.
+not be added to the targets built with that
+construction environment. The first command
+in the action string and the first after any
+<literal>&&</literal> entries will be found
+by searching the <varname>PATH</varname>
+variable in the <varname>ENV</varname>
+environment used to execute the command.
+All other commands will only be found if they
+are absolute paths or valid paths relative
+to the working directory.
+</para>
+
 <example_commands>
 env = Environment(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 </example_commands>
diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py
index 1711de1..c6c19ae 100644
--- a/src/engine/SCons/Tool/JavaCommon.py
+++ b/src/engine/SCons/Tool/JavaCommon.py
@@ -404,7 +404,7 @@ if java_parsing:
 
 
     def parse_java_file(fn, version=default_java_version):
-        with open(fn, 'r') as f:
+        with open(fn, 'r', encoding='utf-8') as f:
             data = f.read()
         return parse_java(data, version)
 
diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py
index 9242624..b0a788e 100644
--- a/src/engine/SCons/Tool/JavaCommonTests.py
+++ b/src/engine/SCons/Tool/JavaCommonTests.py
@@ -68,6 +68,30 @@ public class Foo
         assert classes == ['Foo'], classes
 
 
+    def test_file_parser(self):
+        """Test the file parser"""
+        input = """\
+package com.sub.bar;
+
+public class Foo
+{
+     public static void main(String[] args)
+     {
+        /* This tests that unicde is handled . */
+        String hello1 = new String("ఎత్తువెడల్పు");
+     }
+}
+"""
+        file_name = 'test_file_parser.java'
+        with open(file_name, 'w', encoding='UTF-8') as jf:
+            print(input, file=jf)
+
+        pkg_dir, classes = SCons.Tool.JavaCommon.parse_java_file(file_name)
+        if os.path.exists(file_name):
+            os.remove(file_name)
+        assert pkg_dir == os.path.join('com', 'sub', 'bar'), pkg_dir
+        assert classes == ['Foo'], classes
+
 
     def test_dollar_sign(self):
         """Test class names with $ in them"""
diff --git a/src/engine/SCons/Tool/textfile.py b/src/engine/SCons/Tool/textfile.py
index c233658..b404304 100644
--- a/src/engine/SCons/Tool/textfile.py
+++ b/src/engine/SCons/Tool/textfile.py
@@ -171,7 +171,7 @@ _text_builder = SCons.Builder.Builder(
     suffix='$TEXTFILESUFFIX',
 )
 
-_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'TEXTFILESUFFIX']
+_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'SUBSTFILESUFFIX']
 _subst_builder = SCons.Builder.Builder(
     action=SCons.Action.Action(_action, _strfunc, varlist=_subst_varlist),
     source_factory=SCons.Node.FS.File,
diff --git a/test/Batch/action-changed.py b/test/Batch/action-changed.py
index b2b1673..da5115e 100644
--- a/test/Batch/action-changed.py
+++ b/test/Batch/action-changed.py
@@ -33,7 +33,11 @@ import os
 
 import TestSCons
 
-python = TestSCons.python
+# swap slashes because on py3 on win32 inside the generated build.py
+# the backslashes are getting interpretted as unicode which is
+# invalid.
+python = TestSCons.python.replace('\\','//')
+_python_ = TestSCons._python_
 
 test = TestSCons.TestSCons()
 
@@ -53,17 +57,21 @@ sys.exit(0)
 test.write('build.py', build_py_contents % (python, 'one'))
 os.chmod(test.workpath('build.py'), 0o755)
 
+build_py_workpath = test.workpath('build.py')
+
+# Provide IMPLICIT_COMMAND_DEPENDENCIES=2 so we take a dependency on build.py.
+# Without that, we only scan the first entry in the action string.
 test.write('SConstruct', """
-env = Environment()
+env = Environment(IMPLICIT_COMMAND_DEPENDENCIES=2)
 env.PrependENVPath('PATHEXT', '.PY')
-bb = Action(r'"%s" $CHANGED_TARGETS -- $CHANGED_SOURCES',
+bb = Action(r'%(_python_)s "%(build_py_workpath)s" $CHANGED_TARGETS -- $CHANGED_SOURCES',
             batch_key=True,
             targets='CHANGED_TARGETS')
 env['BUILDERS']['Batch'] = Builder(action=bb)
 env.Batch('f1.out', 'f1.in')
 env.Batch('f2.out', 'f2.in')
 env.Batch('f3.out', 'f3.in')
-""" % test.workpath('build.py'))
+""" % locals())
 
 test.write('f1.in', "f1.in\n")
 test.write('f2.in', "f2.in\n")
diff --git a/test/Repository/Program.py b/test/Repository/Program.py
index 1eb18d8..75860a7 100644
--- a/test/Repository/Program.py
+++ b/test/Repository/Program.py
@@ -32,25 +32,24 @@ if sys.platform == 'win32':
 else:
     _exe = ''
 
-test = TestSCons.TestSCons()
-
-
-
-# First, test a single repository.
-test.subdir('repository', 'work1')
-
-repository = test.workpath('repository')
-repository_foo_c = test.workpath('repository', 'foo.c')
-work1_foo = test.workpath('work1', 'foo' + _exe)
-work1_foo_c = test.workpath('work1', 'foo.c')
-
-test.write(['work1', 'SConstruct'], r"""
+for implicit_deps in ['0', '1', '"all"']:
+    # First, test a single repository.
+    test = TestSCons.TestSCons()
+    test.verbose_set(1)
+    test.subdir('repository', 'work1')
+    repository = test.workpath('repository')
+    repository_foo_c = test.workpath('repository', 'foo.c')
+    work1_foo = test.workpath('work1', 'foo' + _exe)
+    work1_foo_c = test.workpath('work1', 'foo.c')
+
+    test.write(['work1', 'SConstruct'], r"""
+DefaultEnvironment(tools=[])
 Repository(r'%s')
-env = Environment()
+env = Environment(IMPLICIT_COMMAND_DEPENDENCIES=%s)
 env.Program(target= 'foo', source = Split('aaa.c bbb.c foo.c'))
-""" % repository)
+""" % (repository, implicit_deps))
 
-test.write(['repository', 'aaa.c'], r"""
+    test.write(['repository', 'aaa.c'], r"""
 #include <stdio.h>
 void
 aaa(void)
@@ -59,7 +58,7 @@ aaa(void)
 }
 """)
 
-test.write(['repository', 'bbb.c'], r"""
+    test.write(['repository', 'bbb.c'], r"""
 #include <stdio.h>
 void
 bbb(void)
@@ -68,7 +67,7 @@ bbb(void)
 }
 """)
 
-test.write(['repository', 'foo.c'], r"""
+    test.write(['repository', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -85,21 +84,21 @@ main(int argc, char *argv[])
 }
 """)
 
-# Make the entire repository non-writable, so we'll detect
-# if we try to write into it accidentally.
-test.writable('repository', 0)
+    # Make the entire repository non-writable, so we'll detect
+    # if we try to write into it accidentally.
+    test.writable('repository', 0)
 
-test.run(chdir = 'work1', arguments = '.')
+    test.run(chdir = 'work1', arguments = '.')
 
-test.run(program = work1_foo, stdout = """repository/aaa.c
+    test.run(program = work1_foo, stdout = """repository/aaa.c
 repository/bbb.c
 repository/foo.c
 """)
 
-test.up_to_date(chdir = 'work1', arguments = '.')
+    test.up_to_date(chdir = 'work1', arguments = '.')
 
-#
-test.write(['work1', 'bbb.c'], r"""
+    #
+    test.write(['work1', 'bbb.c'], r"""
 #include <stdio.h>
 void
 bbb(void)
@@ -108,17 +107,17 @@ bbb(void)
 }
 """)
 
-test.run(chdir = 'work1', arguments = '.')
+    test.run(chdir = 'work1', arguments = '.')
 
-test.run(program = work1_foo, stdout = """repository/aaa.c
+    test.run(program = work1_foo, stdout = """repository/aaa.c
 work1/bbb.c
 repository/foo.c
 """)
 
-test.up_to_date(chdir = 'work1', arguments = '.')
+    test.up_to_date(chdir = 'work1', arguments = '.')
 
-#
-test.write(['work1', 'aaa.c'], r"""
+    #
+    test.write(['work1', 'aaa.c'], r"""
 #include <stdio.h>
 void
 aaa(void)
@@ -127,7 +126,7 @@ aaa(void)
 }
 """)
 
-test.write(['work1', 'foo.c'], r"""
+    test.write(['work1', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 extern void aaa(void);
@@ -143,57 +142,58 @@ main(int argc, char *argv[])
 }
 """)
 
-test.run(chdir = 'work1', arguments = '.')
+    test.run(chdir = 'work1', arguments = '.')
 
-test.run(program = work1_foo, stdout = """work1/aaa.c
+    test.run(program = work1_foo, stdout = """work1/aaa.c
 work1/bbb.c
 work1/foo.c
 """)
 
-test.up_to_date(chdir = 'work1', arguments = '.')
+    test.up_to_date(chdir = 'work1', arguments = '.')
 
-#
-test.unlink(['work1', 'aaa.c'])
+    #
+    test.unlink(['work1', 'aaa.c'])
 
-test.run(chdir = 'work1', arguments = '.')
+    test.run(chdir = 'work1', arguments = '.')
 
-test.run(program = work1_foo, stdout = """repository/aaa.c
+    test.run(program = work1_foo, stdout = """repository/aaa.c
 work1/bbb.c
 work1/foo.c
 """)
 
-test.up_to_date(chdir = 'work1', arguments = '.')
+    test.up_to_date(chdir = 'work1', arguments = '.')
 
-#
-test.unlink(['work1', 'bbb.c'])
-test.unlink(['work1', 'foo.c'])
+    #
+    test.unlink(['work1', 'bbb.c'])
+    test.unlink(['work1', 'foo.c'])
 
-test.run(chdir = 'work1', arguments = '.')
+    test.run(chdir = 'work1', arguments = '.')
 
-test.run(program = work1_foo, stdout = """repository/aaa.c
+    test.run(program = work1_foo, stdout = """repository/aaa.c
 repository/bbb.c
 repository/foo.c
 """)
 
-test.up_to_date(chdir = 'work1', arguments = '.')
+    test.up_to_date(chdir = 'work1', arguments = '.')
 
 
 
-# Now, test multiple repositories.
-test.subdir('repository.new', 'repository.old', 'work2')
+    # Now, test multiple repositories.
+    test.subdir('repository.new', 'repository.old', 'work2')
 
-repository_new = test.workpath('repository.new')
-repository_old = test.workpath('repository.old')
-work2_foo = test.workpath('work2', 'foo' + _exe)
+    repository_new = test.workpath('repository.new')
+    repository_old = test.workpath('repository.old')
+    work2_foo = test.workpath('work2', 'foo' + _exe)
 
-test.write(['work2', 'SConstruct'], r"""
+    test.write(['work2', 'SConstruct'], r"""
+DefaultEnvironment(tools=[])
 Repository(r'%s')
 Repository(r'%s')
 env = Environment()
 env.Program(target= 'foo', source = Split('aaa.c bbb.c foo.c'))
 """ % (repository_new, repository_old))
 
-test.write(['repository.old', 'aaa.c'], r"""
+    test.write(['repository.old', 'aaa.c'], r"""
 #include <stdio.h>
 void
 aaa(void)
@@ -202,7 +202,7 @@ aaa(void)
 }
 """)
 
-test.write(['repository.old', 'bbb.c'], r"""
+    test.write(['repository.old', 'bbb.c'], r"""
 #include <stdio.h>
 void
 bbb(void)
@@ -211,7 +211,7 @@ bbb(void)
 }
 """)
 
-test.write(['repository.old', 'foo.c'], r"""
+    test.write(['repository.old', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 extern void aaa(void);
@@ -227,24 +227,24 @@ main(int argc, char *argv[])
 }
 """)
 
-# Make both repositories non-writable, so we'll detect
-# if we try to write into it accidentally.
-test.writable('repository.new', 0)
-test.writable('repository.old', 0)
+    # Make both repositories non-writable, so we'll detect
+    # if we try to write into it accidentally.
+    test.writable('repository.new', 0)
+    test.writable('repository.old', 0)
 
-test.run(chdir = 'work2', arguments = '.')
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """repository.old/aaa.c
+    test.run(program = work2_foo, stdout = """repository.old/aaa.c
 repository.old/bbb.c
 repository.old/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
-#
-test.writable('repository.new', 1)
+    #
+    test.writable('repository.new', 1)
 
-test.write(['repository.new', 'aaa.c'], r"""
+    test.write(['repository.new', 'aaa.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -254,7 +254,7 @@ aaa(void)
 }
 """)
 
-test.write(['work2', 'bbb.c'], r"""
+    test.write(['work2', 'bbb.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -264,20 +264,20 @@ bbb(void)
 }
 """)
 
-#
-test.writable('repository.new', 0)
+    #
+    test.writable('repository.new', 0)
 
-test.run(chdir = 'work2', arguments = '.')
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """repository.new/aaa.c
+    test.run(program = work2_foo, stdout = """repository.new/aaa.c
 work2/bbb.c
 repository.old/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
-#
-test.write(['work2', 'aaa.c'], r"""
+    #
+    test.write(['work2', 'aaa.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -287,7 +287,7 @@ aaa(void)
 }
 """)
 
-test.write(['work2', 'foo.c'], r"""
+    test.write(['work2', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 extern void aaa(void);
@@ -303,59 +303,59 @@ main(int argc, char *argv[])
 }
 """)
 
-#
-test.run(chdir = 'work2', arguments = '.')
+    #
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """work2/aaa.c
+    test.run(program = work2_foo, stdout = """work2/aaa.c
 work2/bbb.c
 work2/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
-#
-test.unlink(['work2', 'aaa.c'])
-test.unlink(['work2', 'bbb.c'])
+    #
+    test.unlink(['work2', 'aaa.c'])
+    test.unlink(['work2', 'bbb.c'])
 
-#
-test.run(chdir = 'work2', arguments = '.')
+    #
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """repository.new/aaa.c
+    test.run(program = work2_foo, stdout = """repository.new/aaa.c
 repository.old/bbb.c
 work2/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
-#
-test.unlink(['work2', 'foo.c'])
+    #
+    test.unlink(['work2', 'foo.c'])
 
-#
-test.run(chdir = 'work2', arguments = '.')
+    #
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """repository.new/aaa.c
+    test.run(program = work2_foo, stdout = """repository.new/aaa.c
 repository.old/bbb.c
 repository.old/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
-#
-test.writable('repository.new', 1)
+    #
+    test.writable('repository.new', 1)
 
-test.unlink(['repository.new', 'aaa.c'])
+    test.unlink(['repository.new', 'aaa.c'])
 
-test.writable('repository.new', 0)
+    test.writable('repository.new', 0)
 
-#
-test.run(chdir = 'work2', arguments = '.')
+    #
+    test.run(chdir = 'work2', arguments = '.')
 
-test.run(program = work2_foo, stdout = """repository.old/aaa.c
+    test.run(program = work2_foo, stdout = """repository.old/aaa.c
 repository.old/bbb.c
 repository.old/foo.c
 """)
 
-test.up_to_date(chdir = 'work2', arguments = '.')
+    test.up_to_date(chdir = 'work2', arguments = '.')
 
 
 #
diff --git a/test/Repository/StaticLibrary.py b/test/Repository/StaticLibrary.py
index 4f8160c..29cb841 100644
--- a/test/Repository/StaticLibrary.py
+++ b/test/Repository/StaticLibrary.py
@@ -30,35 +30,37 @@ import TestSCons
 _obj = TestSCons._obj
 _exe = TestSCons._exe
 
-test = TestSCons.TestSCons()
-
-#
-test.subdir('repository', 'work1', 'work2', 'work3')
-
-#
-workpath_repository = test.workpath('repository')
-repository_aaa_obj = test.workpath('repository', 'aaa' + _obj)
-repository_bbb_obj = test.workpath('repository', 'bbb' + _obj)
-repository_foo_obj = test.workpath('repository', 'foo' + _obj)
-repository_foo = test.workpath('repository', 'foo' + _exe)
-work1_foo = test.workpath('work1', 'foo' + _exe)
-work2_aaa_obj = test.workpath('work2', 'aaa' + _obj)
-work2_foo_obj = test.workpath('work2', 'foo' + _obj)
-work2_foo = test.workpath('work2', 'foo' + _exe)
-work3_aaa_obj = test.workpath('work3', 'aaa' + _obj)
-work3_bbb_obj = test.workpath('work3', 'bbb' + _obj)
-work3_foo = test.workpath('work3', 'foo' + _exe)
-
-opts = '-Y ' + workpath_repository
-
-#
-test.write(['repository', 'SConstruct'], """
-env = Environment(LIBS = ['xxx'], LIBPATH = '.')
+for implicit_deps in ['1', '2']:
+    test = TestSCons.TestSCons()
+
+    #
+    test.subdir('repository', 'work1', 'work2', 'work3')
+
+    #
+    workpath_repository = test.workpath('repository')
+    repository_aaa_obj = test.workpath('repository', 'aaa' + _obj)
+    repository_bbb_obj = test.workpath('repository', 'bbb' + _obj)
+    repository_foo_obj = test.workpath('repository', 'foo' + _obj)
+    repository_foo = test.workpath('repository', 'foo' + _exe)
+    work1_foo = test.workpath('work1', 'foo' + _exe)
+    work2_aaa_obj = test.workpath('work2', 'aaa' + _obj)
+    work2_foo_obj = test.workpath('work2', 'foo' + _obj)
+    work2_foo = test.workpath('work2', 'foo' + _exe)
+    work3_aaa_obj = test.workpath('work3', 'aaa' + _obj)
+    work3_bbb_obj = test.workpath('work3', 'bbb' + _obj)
+    work3_foo = test.workpath('work3', 'foo' + _exe)
+
+    opts = '-Y ' + workpath_repository
+
+    #
+    test.write(['repository', 'SConstruct'], """
+env = Environment(LIBS = ['xxx'], LIBPATH = '.',
+                  IMPLICIT_COMMAND_DEPENDENCIES=%s)
 env.Library(target = 'xxx', source = ['aaa.c', 'bbb.c'])
 env.Program(target = 'foo', source = 'foo.c')
-""")
+""" % implicit_deps)
 
-test.write(['repository', 'aaa.c'], r"""
+    test.write(['repository', 'aaa.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -68,7 +70,7 @@ aaa(void)
 }
 """)
 
-test.write(['repository', 'bbb.c'], r"""
+    test.write(['repository', 'bbb.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -78,7 +80,7 @@ bbb(void)
 }
 """)
 
-test.write(['repository', 'foo.c'], r"""
+    test.write(['repository', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 extern void aaa(void);
@@ -94,29 +96,29 @@ main(int argc, char *argv[])
 }
 """)
 
-# Make the repository non-writable,
-# so we'll detect if we try to write into it accidentally.
-test.writable('repository', 0)
+    # Make the repository non-writable,
+    # so we'll detect if we try to write into it accidentally.
+    test.writable('repository', 0)
 
-#
-test.run(chdir = 'work1', options = opts, arguments = ".",
-         stderr=TestSCons.noisy_ar,
-         match=TestSCons.match_re_dotall)
+    #
+    test.run(chdir = 'work1', options = opts, arguments = ".",
+             stderr=TestSCons.noisy_ar,
+             match=TestSCons.match_re_dotall)
 
-test.run(program = work1_foo, stdout =
+    test.run(program = work1_foo, stdout =
 """repository/aaa.c
 repository/bbb.c
 repository/foo.c
 """)
 
-test.fail_test(os.path.exists(repository_aaa_obj))
-test.fail_test(os.path.exists(repository_bbb_obj))
-test.fail_test(os.path.exists(repository_foo_obj))
-test.fail_test(os.path.exists(repository_foo))
+    test.fail_test(os.path.exists(repository_aaa_obj))
+    test.fail_test(os.path.exists(repository_bbb_obj))
+    test.fail_test(os.path.exists(repository_foo_obj))
+    test.fail_test(os.path.exists(repository_foo))
 
-test.up_to_date(chdir = 'work1', options = opts, arguments = ".")
+    test.up_to_date(chdir = 'work1', options = opts, arguments = ".")
 
-test.write(['work1', 'bbb.c'], r"""
+    test.write(['work1', 'bbb.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -126,49 +128,49 @@ bbb(void)
 }
 """)
 
-test.run(chdir = 'work1', options = opts, arguments = ".",
-         stderr=TestSCons.noisy_ar,
-         match=TestSCons.match_re_dotall)
+    test.run(chdir = 'work1', options = opts, arguments = ".",
+             stderr=TestSCons.noisy_ar,
+             match=TestSCons.match_re_dotall)
 
-test.run(program = work1_foo, stdout =
+    test.run(program = work1_foo, stdout =
 """repository/aaa.c
 work1/bbb.c
 repository/foo.c
 """)
 
-test.fail_test(os.path.exists(repository_aaa_obj))
-test.fail_test(os.path.exists(repository_bbb_obj))
-test.fail_test(os.path.exists(repository_foo_obj))
-test.fail_test(os.path.exists(repository_foo))
+    test.fail_test(os.path.exists(repository_aaa_obj))
+    test.fail_test(os.path.exists(repository_bbb_obj))
+    test.fail_test(os.path.exists(repository_foo_obj))
+    test.fail_test(os.path.exists(repository_foo))
 
-test.up_to_date(chdir = 'work1', options = opts, arguments = ".")
+    test.up_to_date(chdir = 'work1', options = opts, arguments = ".")
 
-#
-test.writable('repository', 1)
+    #
+    test.writable('repository', 1)
 
-test.run(chdir = 'repository', options = opts, arguments = ".",
-         stderr=TestSCons.noisy_ar,
-         match=TestSCons.match_re_dotall)
+    test.run(chdir = 'repository', options = opts, arguments = ".",
+             stderr=TestSCons.noisy_ar,
+             match=TestSCons.match_re_dotall)
 
-test.run(program = repository_foo, stdout =
+    test.run(program = repository_foo, stdout =
 """repository/aaa.c
 repository/bbb.c
 repository/foo.c
 """)
 
-test.fail_test(not os.path.exists(repository_aaa_obj))
-test.fail_test(not os.path.exists(repository_bbb_obj))
-test.fail_test(not os.path.exists(repository_foo_obj))
+    test.fail_test(not os.path.exists(repository_aaa_obj))
+    test.fail_test(not os.path.exists(repository_bbb_obj))
+    test.fail_test(not os.path.exists(repository_foo_obj))
 
-test.up_to_date(chdir = 'repository', options = opts, arguments = ".")
+    test.up_to_date(chdir = 'repository', options = opts, arguments = ".")
 
-#
-test.writable('repository', 0)
+    #
+    test.writable('repository', 0)
 
-#
-test.up_to_date(chdir = 'work2', options = opts, arguments = ".")
+    #
+    test.up_to_date(chdir = 'work2', options = opts, arguments = ".")
 
-test.write(['work2', 'bbb.c'], r"""
+    test.write(['work2', 'bbb.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 void
@@ -178,25 +180,25 @@ bbb(void)
 }
 """)
 
-test.run(chdir = 'work2', options = opts, arguments = ".",
-         stderr=TestSCons.noisy_ar,
-         match=TestSCons.match_re_dotall)
+    test.run(chdir = 'work2', options = opts, arguments = ".",
+             stderr=TestSCons.noisy_ar,
+             match=TestSCons.match_re_dotall)
 
-test.run(program = work2_foo, stdout =
+    test.run(program = work2_foo, stdout =
 """repository/aaa.c
 work2/bbb.c
 repository/foo.c
 """)
 
-test.fail_test(os.path.exists(work2_aaa_obj))
-test.fail_test(os.path.exists(work2_foo_obj))
+    test.fail_test(os.path.exists(work2_aaa_obj))
+    test.fail_test(os.path.exists(work2_foo_obj))
 
-test.up_to_date(chdir = 'work2', options = opts, arguments = ".")
+    test.up_to_date(chdir = 'work2', options = opts, arguments = ".")
 
-#
-test.up_to_date(chdir = 'work3', options = opts, arguments = ".")
+    #
+    test.up_to_date(chdir = 'work3', options = opts, arguments = ".")
 
-test.write(['work3', 'foo.c'], r"""
+    test.write(['work3', 'foo.c'], r"""
 #include <stdio.h>
 #include <stdlib.h>
 extern void aaa(void);
@@ -212,18 +214,19 @@ main(int argc, char *argv[])
 }
 """)
 
-test.run(chdir = 'work3', options = opts, arguments = ".")
+    test.run(chdir = 'work3', options = opts, arguments = ".")
 
-test.run(program = work3_foo, stdout =
+    test.run(program = work3_foo, stdout =
 """repository/aaa.c
 repository/bbb.c
 work3/foo.c
 """)
 
-test.fail_test(os.path.exists(work3_aaa_obj))
-test.fail_test(os.path.exists(work3_bbb_obj))
+    test.fail_test(os.path.exists(work3_aaa_obj))
+    test.fail_test(os.path.exists(work3_bbb_obj))
+
+    test.up_to_date(chdir = 'work3', options = opts, arguments = ".")
 
-test.up_to_date(chdir = 'work3', options = opts, arguments = ".")
 
 #
 test.pass_test()
diff --git a/test/implicit/IMPLICIT_COMMAND_DEPENDENCIES.py b/test/implicit/IMPLICIT_COMMAND_DEPENDENCIES.py
index 2431a61..bec2255 100644
--- a/test/implicit/IMPLICIT_COMMAND_DEPENDENCIES.py
+++ b/test/implicit/IMPLICIT_COMMAND_DEPENDENCIES.py
@@ -40,6 +40,8 @@ _python_ = TestSCons._python_
 
 test = TestSCons.TestSCons()
 
+test.write('file.in',     "file.in\n")
+
 generate_build_py_py_contents = """\
 #!%(python)s
 import os
@@ -61,17 +63,17 @@ os.chmod(sys.argv[1], 0o755)
 
 """
 
-extra = ''
-test.write('generate_build_py.py', generate_build_py_py_contents % locals())
-
+workpath = test.workpath()
 test.write('SConstruct', """
 DefaultEnvironment(tools=[])
 generate = Builder(action = r'%(_python_)s $GENERATE $TARGET')
-build = Builder(action = r'$BUILD_PY $TARGET $SOURCES')
+build = Builder(action = r'%(_python_)s $BUILD_PY $TARGET $SOURCES')
+cd_and_build = Builder(action = r'cd %(workpath)s && %(_python_)s $BUILD_PY $TARGET $SOURCES')
 env = Environment(tools=[],
                   BUILDERS = {
                         'GenerateBuild' : generate,
                         'BuildFile' : build,
+                        'CdAndBuildFile': cd_and_build,
                   },
                   GENERATE = 'generate_build_py.py',
                   BUILD_PY = 'build.py',
@@ -80,6 +82,8 @@ env.PrependENVPath('PATH', '.')
 env.PrependENVPath('PATHEXT', '.PY')
 env0        = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 0)
 env1        = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 1)
+env2        = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 2)
+envAll      = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = 'all')
 envNone     = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = None)
 envFalse    = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = False)
 envTrue     = env.Clone(IMPLICIT_COMMAND_DEPENDENCIES = True)
@@ -90,44 +94,59 @@ AlwaysBuild(build_py)
 env.BuildFile('file.out',               'file.in')
 env0.BuildFile('file0.out',             'file.in')
 env1.BuildFile('file1.out',             'file.in')
+env2.BuildFile('file2.out',             'file.in')
+envAll.BuildFile('fileall.out',         'file.in')
 envNone.BuildFile('fileNone.out',       'file.in')
 envFalse.BuildFile('fileFalse.out',     'file.in')
 envTrue.BuildFile('fileTrue.out',       'file.in')
 envTrue.BuildFile('fileQuote.out',      'file.in', BUILD_PY='"build.py"')
-""" % locals())
-
 
+env1.CdAndBuildFile('cd_file1.out',     'file.in')
+env2.CdAndBuildFile('cd_file2.out',     'file.in')
+envAll.CdAndBuildFile('cd_fileall.out', 'file.in')
+""" % locals())
 
-test.write('file.in',     "file.in\n")
 
-test.run(arguments = '--tree=all .')
 
-expect_none = 'build.py %s file.in\nfile.in\n'
+def run_test(extra, python, _python_):
+    # Write the generate_build_py.py file. This uses the contents of the
+    # variable "extra" while writing build.py.
+    test.write('generate_build_py.py',
+               generate_build_py_py_contents % locals())
 
-test.must_match('file.out',         expect_none % 'file.out', mode='r')
-test.must_match('file0.out',        expect_none % 'file0.out', mode='r')
-test.must_match('file1.out',        expect_none % 'file1.out', mode='r')
-test.must_match('fileNone.out',     expect_none % 'fileNone.out', mode='r')
-test.must_match('fileFalse.out',    expect_none % 'fileFalse.out', mode='r')
-test.must_match('fileTrue.out',     expect_none % 'fileTrue.out', mode='r')
-test.must_match('fileQuote.out',    expect_none % 'fileQuote.out', mode='r')
+    # Run the SConscript file.
+    test.run(arguments = '--tree=all .')
 
+    # Generate some expected data of actions involving build.py. This expected
+    # data depends on the value of "extra".
+    build_none = 'build.py %s file.in\nfile.in\n'
+    build_extra = (build_none if not extra else
+        'build.py %s file.in\n{}file.in\n'.format(
+            extra.replace('\\\\n', '\n')))
+    build_extra_abs = '{} %s file.in\n{}file.in\n'.format(
+        test.workpath('build.py'),
+        extra.replace('\\\\n', '\n'))
 
+    empty_none = 'empty.py %s file.in\nfile.in\n'
 
-extra = 'xyzzy\\\\n'
-test.write('generate_build_py.py', generate_build_py_py_contents % locals())
+    # Verify that the output matches what is expected.
+    test.must_match('file.out',       build_none % 'file.out', mode='r')
+    test.must_match('file0.out',      build_none % 'file0.out', mode='r')
+    test.must_match('file1.out',      build_none % 'file1.out', mode='r')
+    test.must_match('file2.out',      build_extra % 'file2.out', mode='r')
+    test.must_match('fileall.out',    build_extra % 'fileall.out', mode='r')
+    test.must_match('fileNone.out',   build_none % 'fileNone.out', mode='r')
+    test.must_match('fileFalse.out',  build_none % 'fileFalse.out', mode='r')
+    test.must_match('fileTrue.out',   build_none % 'fileTrue.out', mode='r')
+    test.must_match('fileQuote.out',  build_none % 'fileQuote.out', mode='r')
+    test.must_match('cd_file1.out',   build_none % 'cd_file1.out', mode='r')
+    test.must_match('cd_file2.out',   build_extra % 'cd_file2.out', mode='r')
+    test.must_match('cd_fileall.out', build_extra % 'cd_fileall.out', mode='r')
 
-test.run(arguments = '--tree=all .')
 
-expect_extra = 'build.py %s file.in\nxyzzy\nfile.in\n'
+run_test('', python, _python_)
+run_test('xyzzy\\\\n', python, _python_)
 
-test.must_match('file.out',         expect_extra % 'file.out', mode='r')
-test.must_match('file0.out',        expect_none % 'file0.out', mode='r')
-test.must_match('file1.out',        expect_extra % 'file1.out', mode='r')
-test.must_match('fileNone.out',     expect_none % 'fileNone.out', mode='r')
-test.must_match('fileFalse.out',    expect_none % 'fileFalse.out', mode='r')
-test.must_match('fileTrue.out',     expect_extra % 'fileTrue.out', mode='r')
-test.must_match('fileQuote.out',    expect_extra % 'fileQuote.out', mode='r')
 
 
 test.pass_test()
diff --git a/test/textfile/fixture/SConstruct.issue-3540 b/test/textfile/fixture/SConstruct.issue-3540
new file mode 100644
index 0000000..021689a
--- /dev/null
+++ b/test/textfile/fixture/SConstruct.issue-3540
@@ -0,0 +1,23 @@
+"""
+Test for GH Issue 3540
+
+textfile()'s action is not sensitive to changes in TEXTFILESUFFIX (rather was sensitive to SUBSTFILESUFFIX)
+
+"""
+
+DefaultEnvironment(tools=[])
+
+text_file_suffix = ARGUMENTS.get('text_file_suffix', 'DEFAULTSUFFIX')
+
+env = Environment(tools=['textfile'],
+                  TEXTFILESUFFIX=text_file_suffix)
+
+env['FOO_PATH'] = "test-value-1"
+
+foo = env.Substfile(
+    target="substfile",
+    source="substfile.in",
+    SUBST_DICT={
+        "@foo_path@": "$FOO_PATH",
+    }
+)
diff --git a/test/textfile/fixture/SConstruct.issue-3550 b/test/textfile/fixture/SConstruct.issue-3550
index 8502717..0239ade 100644
--- a/test/textfile/fixture/SConstruct.issue-3550
+++ b/test/textfile/fixture/SConstruct.issue-3550
@@ -4,8 +4,8 @@ env = Environment(tools=['textfile'])
 env['FOO_PATH'] = r'Z:\mongo\build\install\bin'
 
 foo = env.Substfile(
-    target="foo",
-    source="foo-3550.in",
+    target="substfile",
+    source="substfile.in",
     SUBST_DICT={
         "@foo_path@": "$FOO_PATH",
     }
diff --git a/test/textfile/fixture/foo-3550.in b/test/textfile/fixture/substfile.in
index c806a04..c806a04 100644
--- a/test/textfile/fixture/foo-3550.in
+++ b/test/textfile/fixture/substfile.in
diff --git a/test/textfile/issue-3540.py b/test/textfile/issue-3540.py
new file mode 100644
index 0000000..b53f06a
--- /dev/null
+++ b/test/textfile/issue-3540.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+"""
+Test for GH Issue 3540
+
+textfile()'s action is not sensitive to changes in TEXTFILEPREFIX (rather was sensitive to SUBSTFILEPREFIX)
+
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+# test.verbose_set(1)
+
+test.file_fixture('fixture/substfile.in', 'substfile.in')
+test.file_fixture('fixture/SConstruct.issue-3540', 'SConstruct')
+
+test.run()
+test.must_exist('substfile')
+
+test.up_to_date(options='text_file_suffix=BLAH')
+
+
+test.pass_test()
diff --git a/test/textfile/issue-3550.py b/test/textfile/issue-3550.py
index 60430ed..5951d1b 100644
--- a/test/textfile/issue-3550.py
+++ b/test/textfile/issue-3550.py
@@ -23,21 +23,27 @@
 #
 
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+"""
+Test for GH Issue 3550
+
+ You can't pass a Windows path in a value to be interpolated, 
+ because SCons will try to pass it through re.sub which rejects certain character sequences.
+"""
 
 import TestSCons
 
 test = TestSCons.TestSCons()
 
-test.verbose_set(1)
+# test.verbose_set(1)
 
 match_mode = 'r'
 
 test.file_fixture('fixture/SConstruct.issue-3550', 'SConstruct')
-test.file_fixture('fixture/foo-3550.in', 'foo-3550.in')
+test.file_fixture('fixture/substfile.in', 'substfile.in')
 
 test.run(arguments='.')
 
-test.must_match('foo',
+test.must_match('substfile',
                 r'''foo_path: Z:\mongo\build\install\bin
 ''', mode=match_mode)