Update to mainline.

14 files changed, 300 insertions, 31 deletions

diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py
index 9d856c9..2398c10 100644
--- a/src/engine/SCons/ActionTests.py
+++ b/src/engine/SCons/ActionTests.py
@@ -1241,8 +1241,8 @@ class CommandActionTestCase(unittest.TestCase):
                    (env["foo"], env["bar"])
 
         # The number 1 is there to make sure all args get converted to strings.
-        a = SCons.Action.CommandAction(["|", "$(", "$foo", "|", "$bar",
-                                        "$)", "|", "$baz", 1])
+        a = SCons.Action.CommandAction(["|", "$(", "$foo", "|", "$(", "$bar",
+                                        "$)", "stuff",  "$)", "|", "$baz", 1])
         c = a.get_contents(target=[], source=[],
                            env=Environment(foo = 'FFF', bar = 'BBB',
                                            baz = CmdGen))
@@ -1257,7 +1257,7 @@ class CommandActionTestCase(unittest.TestCase):
         c = a.get_contents(target=DummyNode('ttt'), source = DummyNode('sss'),
                            env=SpecialEnvironment(foo = 'GGG', bar = 'CCC',
                                                   baz = 'ZZZ'))
-        assert c == b'subst_target_source: | $( $foo | $bar $) | $baz 1', c
+        assert c == b'subst_target_source: | $( $foo | $( $bar $) stuff $) | $baz 1', c
 
         # We've discussed using the real target and source names in a
         # CommandAction's signature contents.  This would have have the
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index 60a45e4..4f8e41b 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -167,7 +167,7 @@ def _set_SCANNERS(env, key, value):
 
 def _delete_duplicates(l, keep_last):
     """Delete duplicates from a sequence, keeping the first or last."""
-    seen={}
+    seen=set()
     result=[]
     if keep_last:           # reverse in & out, then keep first
         l.reverse()
@@ -175,7 +175,7 @@ def _delete_duplicates(l, keep_last):
         try:
             if i not in seen:
                 result.append(i)
-                seen[i]=1
+                seen.add(i)
         except TypeError:
             # probably unhashable.  Just keep it.
             result.append(i)
@@ -1983,6 +1983,15 @@ class Base(SubstitutionEnvironment):
             return result
         return self.fs.Dir(s, *args, **kw)
 
+    def PyPackageDir(self, modulename):
+        s = self.subst(modulename)
+        if SCons.Util.is_Sequence(s):
+            result=[]
+            for e in s:
+                result.append(self.fs.PyPackageDir(e))
+            return result
+        return self.fs.PyPackageDir(s)
+
     def NoClean(self, *targets):
         """Tags a target so that it will not be cleaned by -c"""
         tlist = []
diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml
index 92bc21a..ccee68d 100644
--- a/src/engine/SCons/Environment.xml
+++ b/src/engine/SCons/Environment.xml
@@ -2504,6 +2504,29 @@ env.PrependUnique(CCFLAGS = '-g', FOO = ['foo.yyy'])
 </summary>
 </scons_function>
 
+<scons_function name="PyPackageDir">
+<arguments>
+(modulename)
+</arguments>
+<summary>
+<para>
+This returns a Directory Node similar to Dir.
+The python module / package is looked up and if located
+the directory is returned for the location.
+<varname>modulename</varname>
+Is a named python package / module to
+lookup the directory for it's location.
+</para>
+<para>
+If
+<varname>modulename</varname>
+is a list, SCons returns a list of Dir nodes.
+Construction variables are expanded in
+<varname>modulename</varname>.
+</para>
+</summary>
+</scons_function>
+
 <scons_function name="Replace">
 <arguments signature="env">
 (key=val, [...])
diff --git a/src/engine/SCons/EnvironmentValues.py b/src/engine/SCons/EnvironmentValues.py
new file mode 100644
index 0000000..e2efccb
--- /dev/null
+++ b/src/engine/SCons/EnvironmentValues.py
@@ -0,0 +1,97 @@
+import re
+
+_is_valid_var = re.compile(r'[_a-zA-Z]\w*$')
+
+_rm = re.compile(r'\$[()]')
+_remove = re.compile(r'\$\([^\$]*(\$[^\)][^\$]*)*\$\)')
+
+# Regular expressions for splitting strings and handling substitutions,
+# for use by the scons_subst() and scons_subst_list() functions:
+#
+# The first expression compiled matches all of the $-introduced tokens
+# that we need to process in some way, and is used for substitutions.
+# The expressions it matches are:
+#
+#       "$$"
+#       "$("
+#       "$)"
+#       "$variable"             [must begin with alphabetic or underscore]
+#       "${any stuff}"
+#
+# The second expression compiled is used for splitting strings into tokens
+# to be processed, and it matches all of the tokens listed above, plus
+# the following that affect how arguments do or don't get joined together:
+#
+#       "   "                   [white space]
+#       "non-white-space"       [without any dollar signs]
+#       "$"                     [single dollar sign]
+#
+_dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
+_dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
+_separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str)
+
+# This regular expression is used to replace strings of multiple white
+# space characters in the string result from the scons_subst() function.
+_space_sep = re.compile(r'[\t ]+(?![^{]*})')
+
+class ValueTypes(object):
+    """
+    Enum to store what type of value the variable holds.
+    """
+    UNKNOWN = 0
+    STRING = 1
+    CALLABLE = 2
+    VARIABLE = 3
+
+
+class EnvironmentValue(object):
+    """
+    Hold a single value. We're going to cache parsed version of the file
+    We're going to keep track of variables which feed into this values evaluation
+    """
+    def __init__(self, value):
+        self.value = value
+        self.var_type = ValueTypes.UNKNOWN
+
+        if callable(self.value):
+            self.var_type = ValueTypes.CALLABLE
+        else:
+            self.parse_value()
+
+
+    def parse_value(self):
+        """
+        Scan the string and break into component values
+        """
+
+        try:
+            if '$' not in self.value:
+                self._parsed = self.value
+                self.var_type = ValueTypes.STRING
+            else:
+                # Now we need to parse the specified string
+                result = _dollar_exps.sub(sub_match, args)
+                print(result)
+            pass
+        except TypeError:
+            # likely callable? either way we don't parse
+            self._parsed = self.value
+
+    def parse_trial(self):
+        """
+        Try alternate parsing methods.
+        :return:
+        """
+        parts = []
+        for c in self.value:
+
+
+
+class EnvironmentValues(object):
+    """
+    A class to hold all the environment variables
+    """
+    def __init__(self, **kw):
+        self._dict = {}
+        for k in kw:
+            self._dict[k] = EnvironmentValue(kw[k])
diff --git a/src/engine/SCons/EnvironmentValuesTest.py b/src/engine/SCons/EnvironmentValuesTest.py
new file mode 100644
index 0000000..58ee9cf
--- /dev/null
+++ b/src/engine/SCons/EnvironmentValuesTest.py
@@ -0,0 +1,16 @@
+import unittest
+
+from  SCons.EnvironmentValues import EnvironmentValues
+
+class MyTestCase(unittest.TestCase):
+    def test_simple_environmentValues(self):
+        """Test comparing SubstitutionEnvironments
+        """
+
+        env1 = EnvironmentValues(XXX='x')
+        env2 = EnvironmentValues(XXX='x',XX="$X", X1="${X}", X2="$($X$)")
+
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index d98f7d0..8c1161d 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -1390,6 +1390,35 @@ class FS(LocalFS):
             if not isinstance(d, SCons.Node.Node):
                 d = self.Dir(d)
             self.Top.addRepository(d)
+    
+    def PyPackageDir(self, modulename):
+        """Locate the directory of a given python module name
+		
+        For example scons might resolve to
+        Windows: C:\Python27\Lib\site-packages\scons-2.5.1
+        Linux: /usr/lib/scons
+
+        This can be useful when we want to determine a toolpath based on a python module name"""
+
+        dirpath = ''
+        if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] in (0,1,2,3,4)):
+            # Python2 Code
+            import imp
+            splitname = modulename.split('.')
+            srchpths = sys.path
+            for item in splitname:
+                file, path, desc = imp.find_module(item, srchpths)
+                if file is not None:
+                    path = os.path.dirname(path)
+                srchpths = [path]
+            dirpath = path
+        else:
+            # Python3 Code
+            import importlib.util
+            modspec = importlib.util.find_spec(modulename)
+            dirpath = os.path.dirname(modspec.origin)
+        return self._lookup(dirpath, None, Dir, True)
+
 
     def variant_dir_target_climb(self, orig, dir, tail):
         """Create targets in corresponding variant directories
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index 3fa3a48..5bdd63e 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -337,6 +337,7 @@ GlobalDefaultEnvironmentFunctions = [
     'Local',
     'ParseDepends',
     'Precious',
+    'PyPackageDir',
     'Repository',
     'Requires',
     'SConsignFile',
diff --git a/src/engine/SCons/Subst.py b/src/engine/SCons/Subst.py
index 9aa4bbc..a68b54d 100644
--- a/src/engine/SCons/Subst.py
+++ b/src/engine/SCons/Subst.py
@@ -338,24 +338,28 @@ SUBST_RAW = 1
 SUBST_SIG = 2
 
 _rm = re.compile(r'\$[()]')
-_remove = re.compile(r'\$\([^\$]*(\$[^\)][^\$]*)*\$\)')
+_rm_split = re.compile(r'(\$[()])')
 
 # Indexed by the SUBST_* constants above.
-_regex_remove = [ _rm, None, _remove ]
+_regex_remove = [ _rm, None, _rm_split ]
 
 def _rm_list(list):
     return [l for l in list if not l in ('$(', '$)')]
 
 def _remove_list(list):
     result = []
-    do_append = result.append
+    depth = 0
     for l in list:
         if l == '$(':
-            do_append = lambda x: None
+            depth += 1
         elif l == '$)':
-            do_append = result.append
-        else:
-            do_append(l)
+            depth -= 1
+            if depth < 0:
+                break
+        elif depth == 0:
+            result.append(l)
+    if depth != 0:
+        return None
     return result
 
 # Indexed by the SUBST_* constants above.
@@ -562,12 +566,19 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={
     except KeyError:
         pass
 
+    res = result
     if is_String(result):
         # Remove $(-$) pairs and any stuff in between,
         # if that's appropriate.
         remove = _regex_remove[mode]
         if remove:
-            result = remove.sub('', result)
+            if mode == SUBST_SIG:
+                result = _list_remove[mode](remove.split(result))
+                if result is None:
+                    raise SCons.Errors.UserError("Unbalanced $(/$) in: " + res)
+                result = ' '.join(result)
+            else:
+                result = remove.sub('', result)
         if mode != SUBST_RAW:
             # Compress strings of white space characters into
             # a single space.
@@ -576,6 +587,8 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={
         remove = _list_remove[mode]
         if remove:
             result = remove(result)
+            if result is None:
+                raise SCons.Errors.UserError("Unbalanced $(/$) in: " + str(res))
 
     return result
 
diff --git a/src/engine/SCons/SubstTests.py b/src/engine/SCons/SubstTests.py
index c11f247..6604128 100644
--- a/src/engine/SCons/SubstTests.py
+++ b/src/engine/SCons/SubstTests.py
@@ -183,6 +183,9 @@ class SubstTestCase(unittest.TestCase):
         'HHH'       : 'III',
         'FFFIII'    : 'BADNEWS',
 
+        'THING1'    : "$(STUFF$)",
+        'THING2'    : "$THING1",
+
         'LITERAL'   : TestLiteral("$XXX"),
 
         # Test that we can expand to and return a function.
@@ -405,6 +408,11 @@ class scons_subst_TestCase(SubstTestCase):
             "test",
             "test",
 
+        "test $( $THING2 $)",
+            "test $( $(STUFF$) $)",
+            "test STUFF",
+            "test",
+
         "$AAA ${AAA}A $BBBB $BBB",
             "a aA  b",
             "a aA b",
@@ -544,6 +552,23 @@ class scons_subst_TestCase(SubstTestCase):
         else:
             raise AssertionError("did not catch expected UserError")
 
+    def test_subst_balance_errors(self):
+        """Test scons_subst():  handling syntax errors"""
+        env = DummyEnv(self.loc)
+        try:
+            scons_subst('$(', env, mode=SUBST_SIG)
+        except SCons.Errors.UserError as e:
+            assert str(e) == "Unbalanced $(/$) in: $(", str(e)
+        else:
+            raise AssertionError("did not catch expected UserError")
+
+        try:
+            scons_subst('$)', env, mode=SUBST_SIG)
+        except SCons.Errors.UserError as e:
+            assert str(e) == "Unbalanced $(/$) in: $)", str(e)
+        else:
+            raise AssertionError("did not catch expected UserError")
+
     def test_subst_type_errors(self):
         """Test scons_subst():  handling type errors"""
         env = DummyEnv(self.loc)
diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py
index a846cfc..b60cd5b 100644
--- a/src/engine/SCons/Tool/MSCommon/common.py
+++ b/src/engine/SCons/Tool/MSCommon/common.py
@@ -130,6 +130,13 @@ def normalize_env(env, keys, force=False):
     if sys32_dir not in normenv['PATH']:
         normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_dir
 
+    # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized"
+    # error starting with Visual Studio 2017, although the script still
+    # seems to work anyway.
+    sys32_wbem_dir = os.path.join(sys32_dir, 'Wbem')
+    if sys32_wbem_dir not in normenv['PATH']:
+        normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir
+
     debug("PATH: %s"%normenv['PATH'])
 
     return normenv
diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py
index 4126314..7c9eab5 100644
--- a/src/engine/SCons/Tool/MSCommon/vc.py
+++ b/src/engine/SCons/Tool/MSCommon/vc.py
@@ -37,6 +37,7 @@ __doc__ = """Module for Visual C/C++ detection and configuration.
 import SCons.compat
 import SCons.Util
 
+import subprocess
 import os
 import platform
 from string import digits as string_digits
@@ -135,9 +136,11 @@ def get_host_target(env):
 
 # If you update this, update SupportedVSList in Tool/MSCommon/vs.py, and the
 # MSVC_VERSION documentation in Tool/msvc.xml.
-_VCVER = ["14.0", "14.0Exp", "12.0", "12.0Exp", "11.0", "11.0Exp", "10.0", "10.0Exp", "9.0", "9.0Exp","8.0", "8.0Exp","7.1", "7.0", "6.0"]
+_VCVER = ["14.1", "14.0", "14.0Exp", "12.0", "12.0Exp", "11.0", "11.0Exp", "10.0", "10.0Exp", "9.0", "9.0Exp","8.0", "8.0Exp","7.1", "7.0", "6.0"]
 
 _VCVER_TO_PRODUCT_DIR = {
+    '14.1' : [
+        (SCons.Util.HKEY_LOCAL_MACHINE, r'')], # Visual Studio 2017 doesn't set this registry key anymore
     '14.0' : [
         (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VisualStudio\14.0\Setup\VC\ProductDir')],
     '14.0Exp' : [
@@ -222,6 +225,33 @@ def is_host_target_supported(host_target, msvc_version):
 
     return True
 
+
+def find_vc_pdir_vswhere(msvc_version):
+    """
+    Find the MSVC product directory using vswhere.exe .
+    Run it asking for specified version and get MSVS  install location
+    :param msvc_version:
+    :return: MSVC install dir
+    """
+    vswhere_path = os.path.join(
+        'C:\\',
+        'Program Files (x86)',
+        'Microsoft Visual Studio',
+        'Installer',
+        'vswhere.exe'
+    )
+    vswhere_cmd = [vswhere_path, '-version', msvc_version, '-property', 'installationPath']
+
+    if os.path.exists(vswhere_path):
+        sp = subprocess.Popen(vswhere_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        vsdir, err = sp.communicate()
+        vc_pdir = os.path.join(vsdir.rstrip(), 'VC')
+        return vc_pdir
+    else:
+        # No vswhere on system, no install info available
+        return None
+
+
 def find_vc_pdir(msvc_version):
     """Try to find the product directory for the given
     version.
@@ -240,16 +270,22 @@ def find_vc_pdir(msvc_version):
     for hkroot, key in hkeys:
         try:
             comps = None
-            if common.is_win64():
-                try:
-                    # ordinally at win64, try Wow6432Node first.
-                    comps = common.read_reg(root + 'Wow6432Node\\' + key, hkroot)
-                except SCons.Util.WinError as e:
-                    # at Microsoft Visual Studio for Python 2.7, value is not in Wow6432Node
-                    pass
-            if not comps:
-                # not Win64, or Microsoft Visual Studio for Python 2.7
-                comps = common.read_reg(root + key, hkroot)
+            if not key:
+                comps = find_vc_pdir_vswhere(msvc_version)
+                if not comps:
+                    debug('find_vc_dir(): no VC found via vswhere for version {}'.format(repr(key)))
+                    raise SCons.Util.WinError
+            else:
+                if common.is_win64():
+                    try:
+                        # ordinally at win64, try Wow6432Node first.
+                        comps = common.read_reg(root + 'Wow6432Node\\' + key, hkroot)
+                    except SCons.Util.WinError as e:
+                        # at Microsoft Visual Studio for Python 2.7, value is not in Wow6432Node
+                        pass
+                if not comps:
+                    # not Win64, or Microsoft Visual Studio for Python 2.7
+                    comps = common.read_reg(root + key, hkroot)
         except SCons.Util.WinError as e:
             debug('find_vc_dir(): no VC registry key {}'.format(repr(key)))
         else:
@@ -281,8 +317,10 @@ def find_batch_file(env,msvc_version,host_arch,target_arch):
     elif vernum < 7:
         pdir = os.path.join(pdir, "Bin")
         batfilename = os.path.join(pdir, "vcvars32.bat")
-    else: # >= 8
+    elif 8 <= vernum <= 14:
         batfilename = os.path.join(pdir, "vcvarsall.bat")
+    else:  # vernum >= 14.1  VS2017 and above
+        batfilename = os.path.join(pdir, "Auxiliary", "Build", "vcvarsall.bat")
 
     if not os.path.exists(batfilename):
         debug("Not found: %s" % batfilename)
diff --git a/src/engine/SCons/Tool/MSCommon/vs.py b/src/engine/SCons/Tool/MSCommon/vs.py
index cff05d5..f9382fb 100644
--- a/src/engine/SCons/Tool/MSCommon/vs.py
+++ b/src/engine/SCons/Tool/MSCommon/vs.py
@@ -199,6 +199,17 @@ class VisualStudio(object):
 # Tool/MSCommon/vc.py, and the MSVC_VERSION documentation in Tool/msvc.xml.
 
 SupportedVSList = [
+    # Visual Studio 2017
+    VisualStudio('14.1',
+                 vc_version='14.1',
+                 sdk_version='10.0A',
+                 hkeys=[],
+                 common_tools_var='VS150COMNTOOLS',
+                 executable_path=r'Common7\IDE\devenv.com',
+                 batch_file_path=r'VC\Auxiliary\Build\vsvars32.bat',
+                 supported_arch=['x86', 'amd64', "arm"],
+                 ),
+
     # Visual Studio 2015
     VisualStudio('14.0',
                  vc_version='14.0',
diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py
index 1ab43e7..6408f8b 100644
--- a/src/engine/SCons/Tool/__init__.py
+++ b/src/engine/SCons/Tool/__init__.py
@@ -1172,12 +1172,12 @@ def tool_list(platform, env):
         ars = ['ar', 'mslib']
     else:
         "prefer GNU tools on all other platforms"
-        linkers = ['gnulink', 'mslink', 'ilink']
-        c_compilers = ['gcc', 'msvc', 'intelc', 'icc', 'cc']
-        cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'cxx']
+        linkers = ['gnulink', 'ilink']
+        c_compilers = ['gcc',  'intelc', 'icc', 'cc']
+        cxx_compilers = ['g++', 'intelc', 'icc', 'cxx']
         assemblers = ['gas', 'nasm', 'masm']
         fortran_compilers = ['gfortran', 'g77', 'ifort', 'ifl', 'f95', 'f90', 'f77']
-        ars = ['ar', 'mslib']
+        ars = ['ar',]
 
     if not str(platform) == 'win32':
         other_plat_tools += ['m4', 'rpm']
diff --git a/src/engine/SCons/Tool/gxx.py b/src/engine/SCons/Tool/gxx.py
index c5eb579..574fd8e 100644
--- a/src/engine/SCons/Tool/gxx.py
+++ b/src/engine/SCons/Tool/gxx.py
@@ -41,7 +41,7 @@ import SCons.Tool
 import SCons.Util
 
 from . import gcc
-cplusplus = __import__(__package__+'.c++', globals(), locals(), ['*'])
+from . import cxx
 
 compilers = ['g++']
 
@@ -52,7 +52,7 @@ def generate(env):
     if 'CXX' not in env:
         env['CXX']    = env.Detect(compilers) or compilers[0]
 
-    cplusplus.generate(env)
+    cxx.generate(env)
 
     # platform specific settings
     if env['PLATFORM'] == 'aix':