From 04b49967dbcd9930087471a1771939b93bfb4e38 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 29 Oct 2018 16:13:32 -0600 Subject: For PR#3222: improve ms build finding Broaden the search to also include Build Tools (the compiler without the whole Visual Studio works). Also in the initial search to see if a suite is valid or not, don't just look for a couple of locations within a given path, do a search. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 3 +++ src/engine/SCons/Tool/MSCommon/vc.py | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 97c3455..a85f37c 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -176,6 +176,9 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Update (pep8) configure-cache script, add a --show option. - Fix for a couple of "what if tool not found" exceptions in framework. - Add Textfile/Substfile to default environment. (issue #3147) + - Improve finding of Microsoft compiler: add a 'products' wildcard + in case Build Tools only is installed; search for cl.exe in located + path instead of just looking for a couple of built-in locations. From Bernhard M. Wiedemann: - Update SCons' internal scons build logic to allow overriding build date diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index 32ee96f..c4b9773 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -241,7 +241,7 @@ def find_vc_pdir_vswhere(msvc_version): 'Installer', 'vswhere.exe' ) - vswhere_cmd = [vswhere_path, '-version', msvc_version, '-property', 'installationPath'] + vswhere_cmd = [vswhere_path, '-products', '*', '-version', msvc_version, '-property', 'installationPath'] if os.path.exists(vswhere_path): sp = subprocess.Popen(vswhere_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -355,15 +355,30 @@ def cached_get_installed_vcs(): def get_installed_vcs(): installed_versions = [] + + def clfind(name, path): + '''Search for a filename in a given path. + + Look for a filename (normally cl.exe) underneath a path. No need + to return the path found, someplace else will dig deeper, this is + just used to confirm a given suite contains that file. Note it + does not promise the cl.exe is the combination of host/target we + actually need, that is also done elsewhere. + ''' + for root, _, files in os.walk(path): + if name in files: + debug('get_installed_vcs cl.exe found %s' % os.path.join(root, name)) + return True + return False + for ver in _VCVER: debug('trying to find VC %s' % ver) try: VC_DIR = find_vc_pdir(ver) if VC_DIR: debug('found VC %s' % ver) - # check to see if the x86 or 64 bit compiler is in the bin dir - if (os.path.exists(os.path.join(VC_DIR, r'bin\cl.exe')) - or os.path.exists(os.path.join(VC_DIR, r'bin\amd64\cl.exe'))): + # now make sure there's a cl.exe in that path + if clfind('cl.exe', VC_DIR): installed_versions.append(ver) else: debug('find_vc_pdir no cl.exe found %s' % ver) -- cgit v0.12 From be0effcc25ebfbb90f6c6e3a40fc4ffebbc0b9aa Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 30 Oct 2018 09:21:16 -0600 Subject: Documentation cleanup for vswhere-related fix Polish up a few docstrings and be more descriptive about the search for cl.exe. Also add requested version qualifiers to entry in CHANGES.txt. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 6 ++- src/engine/SCons/Tool/MSCommon/vc.py | 81 +++++++++++++++++++++++------------- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index a85f37c..6d6e24d 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -177,8 +177,10 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Fix for a couple of "what if tool not found" exceptions in framework. - Add Textfile/Substfile to default environment. (issue #3147) - Improve finding of Microsoft compiler: add a 'products' wildcard - in case Build Tools only is installed; search for cl.exe in located - path instead of just looking for a couple of built-in locations. + in case 2017 Build Tools only is installed as it is considered a separate + product from the default Visual Studio; search for cl.exe in located + path instead of just looking for a couple of built-in locations as + 2017 products are putting cl.exe deeper down the heirarchy. From Bernhard M. Wiedemann: - Update SCons' internal scons build logic to allow overriding build date diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index c4b9773..437633a 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -202,21 +202,21 @@ def msvc_version_to_maj_min(msvc_version): raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) def is_host_target_supported(host_target, msvc_version): - """Return True if the given (host, target) tuple is supported given the - msvc version. - - Parameters - ---------- - host_target: tuple - tuple of (canonalized) host-target, e.g. ("x86", "amd64") for cross - compilation from 32 bits windows to 64 bits. - msvc_version: str - msvc version (major.minor, e.g. 10.0) - - Note - ---- - This only check whether a given version *may* support the given (host, - target), not that the toolchain is actually present on the machine. + """Check if the given (host, target) tuple is supported for given version. + + Args: + host_target: tuple + tuple of (canonalized) host-targets, e.g. ("x86", "amd64") + for cross compilation from 32 bit Windows to 64 bits. + msvc_version: str + msvc version (major.minor, e.g. 10.0) + + Returns: + bool: + + Note: + This only checks whether a given version *may* support the given (host, + target), not that the toolchain is actually present on the machine. """ # We assume that any Visual Studio version supports x86 as a target if host_target[1] != "x86": @@ -229,10 +229,11 @@ def is_host_target_supported(host_target, msvc_version): def find_vc_pdir_vswhere(msvc_version): """ - Find the MSVC product directory using vswhere.exe . + Find the MSVC product directory using vswhere.exe. + Run it asking for specified version and get MSVS install location :param msvc_version: - :return: MSVC install dir + :return: MSVC install dir or None """ vswhere_path = os.path.join( 'C:\\', @@ -256,13 +257,25 @@ def find_vc_pdir_vswhere(msvc_version): def find_vc_pdir(msvc_version): - """Try to find the product directory for the given - version. + """Try to find the product directory for the given version. + + Tries to look up the path using a registry key from the table + _VCVER_TO_PRODUCT_DIR; if there is no key, calls find_vc_pdir_wshere + for help instead. + + Args: + msvc_version: str + msvc version (major.minor, e.g. 10.0) + + Returns: + str: Path found in registry, or None - Note - ---- - If for some reason the requested version could not be found, an - exception which inherits from VisualCException will be raised.""" + Raises: + UnsupportedVersion: if the version is not known by this file. + MissingConfiguration: found version but the directory is missing. + + Both exceptions inherit from VisualCException. + """ root = 'Software\\' try: hkeys = _VCVER_TO_PRODUCT_DIR[msvc_version] @@ -354,16 +367,24 @@ def cached_get_installed_vcs(): return __INSTALLED_VCS_RUN def get_installed_vcs(): + '''Query which versions of compiler suites are installed. + + Returns: + list: version strings from _VCVER that appear to be installed. + ''' installed_versions = [] def clfind(name, path): '''Search for a filename in a given path. - Look for a filename (normally cl.exe) underneath a path. No need + Look for 'name' (normally cl.exe) recursively in 'path'. No need to return the path found, someplace else will dig deeper, this is - just used to confirm a given suite contains that file. Note it - does not promise the cl.exe is the combination of host/target we - actually need, that is also done elsewhere. + just used to confirm a given suite from _VCVER contains that file. + Note it does not promise the cl.exe is the combination of + host/target we actually need, that is also done elsewhere. + + Returns: + bool: ''' for root, _, files in os.walk(path): if name in files: @@ -496,7 +517,7 @@ def msvc_find_valid_batch_script(env,version): (host_target, version) SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) arg = _HOST_TARGET_ARCH_TO_BAT_ARCH[host_target] - + # Get just version numbers maj, min = msvc_version_to_maj_min(version) # VS2015+ @@ -587,11 +608,11 @@ def msvc_setup_env(env): for k, v in d.items(): debug('vc.py:msvc_setup_env() env:%s -> %s'%(k,v)) env.PrependENVPath(k, v, delete_existing=True) - + # final check to issue a warning if the compiler is not present msvc_cl = find_program_path(env, 'cl') if not msvc_cl: - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, "Could not find MSVC compiler 'cl.exe', it may need to be installed separately with Visual Studio") def msvc_exists(version=None): -- cgit v0.12 From eaa2d2712d91d171a79d5dd6528514b13f0dc42f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 18 Nov 2018 14:35:51 -0700 Subject: Typo fix Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 6d6e24d..7b4dc2c 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -180,7 +180,7 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE in case 2017 Build Tools only is installed as it is considered a separate product from the default Visual Studio; search for cl.exe in located path instead of just looking for a couple of built-in locations as - 2017 products are putting cl.exe deeper down the heirarchy. + 2017 products are putting cl.exe deeper down the hierarchy. From Bernhard M. Wiedemann: - Update SCons' internal scons build logic to allow overriding build date -- cgit v0.12 From 9cc5ed21141c76a91a1c287965050071fd5f40fb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 22 Nov 2018 13:48:15 -0700 Subject: PR#3230: fix some debug, accept list from vswhere vc.py: Some of the debug prints weren't quite right, and added a few. When vswhere is called, it can return multiple lines if there are multiple products that match, so handle that case. Preparing for ARM support, add some host/target combos to the table - currently commented out. arm/arm64 added to the canonicalize table where it won't do any harm. common.py: in case we eventually switch to more general logging, use a specific logger rather than the root logger. Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 2 +- src/engine/SCons/Tool/MSCommon/vc.py | 49 +++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b60cd5b..a0140c6 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -46,7 +46,7 @@ elif LOGFILE: debug = lambda message: open(LOGFILE, 'a').write(message + '\n') else: logging.basicConfig(filename=LOGFILE, level=logging.DEBUG) - debug = logging.debug + debug = logging.getLogger(name=__name__).debug else: debug = lambda x: None diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index 437633a..87b6b32 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -80,15 +80,17 @@ _ARCH_TO_CANONICAL = { "i486" : "x86", "i586" : "x86", "i686" : "x86", - "ia64" : "ia64", - "itanium" : "ia64", + "ia64" : "ia64", # deprecated + "itanium" : "ia64", # deprecated "x86" : "x86", "x86_64" : "amd64", "x86_amd64" : "x86_amd64", # Cross compile to 64 bit from 32bits + "arm" : "arm", + "arm64" : "arm64", } -# Given a (host, target) tuple, return the argument for the bat file. Both host -# and targets should be canonalized. +# Given a (host, target) tuple, return the argument for the bat file. +# Both host and targets should be canonalized. _HOST_TARGET_ARCH_TO_BAT_ARCH = { ("x86", "x86"): "x86", ("x86", "amd64"): "x86_amd64", @@ -96,7 +98,12 @@ _HOST_TARGET_ARCH_TO_BAT_ARCH = { ("amd64", "x86_amd64"): "x86_amd64", # This is present in (at least) VS2012 express ("amd64", "amd64"): "amd64", ("amd64", "x86"): "x86", - ("x86", "ia64"): "x86_ia64" + ("x86", "ia64"): "x86_ia64", # gone since 14.0 + #("arm", "arm"): "arm", # since 14.0, maybe gone 14.1? + #("x86", "arm"): "x86_arm", # since 14.0 + #("x86", "arm64"): "x86_arm64", # since 14.1 + #("amd64", "arm"): "amd64_arm", # since 14.0 + #("amd64", "arm64"): "amd64_arm64", # since 14.1 } def get_host_target(env): @@ -247,9 +254,12 @@ def find_vc_pdir_vswhere(msvc_version): if os.path.exists(vswhere_path): sp = subprocess.Popen(vswhere_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) vsdir, err = sp.communicate() - vsdir = vsdir.decode("mbcs") - vsdir = vsdir.rstrip() - vc_pdir = os.path.join(vsdir, 'VC') + vsdir = vsdir.decode("mbcs").splitlines() + # vswhere could easily return multiple lines + # we could define a way to pick the one we prefer, but since + # this data is currently only used to make a check for existence, + # returning the first hit should be good enough for now. + vc_pdir = os.path.join(vsdir[0], 'VC') return vc_pdir else: # No vswhere on system, no install info available @@ -257,7 +267,7 @@ def find_vc_pdir_vswhere(msvc_version): def find_vc_pdir(msvc_version): - """Try to find the product directory for the given version. + """Find the product directory for the given version. Tries to look up the path using a registry key from the table _VCVER_TO_PRODUCT_DIR; if there is no key, calls find_vc_pdir_wshere @@ -289,8 +299,10 @@ def find_vc_pdir(msvc_version): if not key: comps = find_vc_pdir_vswhere(msvc_version) if not comps: - debug('find_vc_dir(): no VC found via vswhere for version {}'.format(repr(key))) + debug('find_vc_pdir_vswhere(): no VC found for version {}'.format(repr(msvc_version))) raise SCons.Util.WinError + debug('find_vc_pdir_vswhere(): VC found: {}'.format(repr(msvc_version))) + return comps else: if common.is_win64(): try: @@ -322,7 +334,7 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): if pdir is None: raise NoVersionFound("No version of Visual Studio found") - debug('vc.py: find_batch_file() pdir:{}'.format(pdir)) + debug('vc.py: find_batch_file() in {}'.format(pdir)) # filter out e.g. "Exp" from the version name msvc_ver_numeric = ''.join([x for x in msvc_version if x in string_digits + "."]) @@ -483,19 +495,19 @@ def msvc_setup_env_once(env): env["MSVC_SETUP_RUN"] = True def msvc_find_valid_batch_script(env,version): - debug('vc.py:msvc_find_valid_batch_script()') # Find the host platform, target platform, and if present the requested # target platform - (host_platform, target_platform,req_target_platform) = get_host_target(env) + platforms = get_host_target(env) + debug("vc.py: msvs_find_valid_batch_script(): host_platform %s, target_platform %s req_target_platform:%s" % platforms) + host_platform, target_platform, req_target_platform) = platforms try_target_archs = [target_platform] - debug("msvs_find_valid_batch_script(): req_target_platform %s target_platform:%s"%(req_target_platform,target_platform)) # VS2012 has a "cross compile" environment to build 64 bit # with x86_amd64 as the argument to the batch setup script - if req_target_platform in ('amd64','x86_64'): + if req_target_platform in ('amd64', 'x86_64'): try_target_archs.append('x86_amd64') - elif not req_target_platform and target_platform in ['amd64','x86_64']: + elif not req_target_platform and target_platform in ['amd64', 'x86_64']: # There may not be "native" amd64, but maybe "cross" x86_amd64 tools try_target_archs.append('x86_amd64') # If the user hasn't specifically requested a TARGET_ARCH, and @@ -542,9 +554,11 @@ def msvc_find_valid_batch_script(env,version): # Try to use the located batch file for this host/target platform combo debug('vc.py:msvc_find_valid_batch_script() use_script 2 %s, args:%s\n' % (repr(vc_script), arg)) + found = None if vc_script: try: d = script_env(vc_script, args=arg) + found = vc_script except BatchFileExecutionError as e: debug('vc.py:msvc_find_valid_batch_script() use_script 3: failed running VC script %s: %s: Error:%s'%(repr(vc_script),arg,e)) vc_script=None @@ -553,6 +567,7 @@ def msvc_find_valid_batch_script(env,version): debug('vc.py:msvc_find_valid_batch_script() use_script 4: trying sdk script: %s'%(sdk_script)) try: d = script_env(sdk_script) + found = sdk_script except BatchFileExecutionError as e: debug('vc.py:msvc_find_valid_batch_script() use_script 5: failed running SDK script %s: Error:%s'%(repr(sdk_script),e)) continue @@ -560,7 +575,7 @@ def msvc_find_valid_batch_script(env,version): debug('vc.py:msvc_find_valid_batch_script() use_script 6: Neither VC script nor SDK script found') continue - debug("vc.py:msvc_find_valid_batch_script() Found a working script/target: %s %s"%(repr(sdk_script),arg)) + debug("vc.py:msvc_find_valid_batch_script() Found a working script/target: %s/%s"%(repr(found),arg)) break # We've found a working target_platform, so stop looking # If we cannot find a viable installed compiler, reset the TARGET_ARCH -- cgit v0.12 From 070d31019e7ed0b57c9670e95db91afcc3932e40 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 22 Nov 2018 14:03:35 -0700 Subject: stray close-paren character Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/vc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index 87b6b32..1f5e2e5 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -500,7 +500,7 @@ def msvc_find_valid_batch_script(env,version): platforms = get_host_target(env) debug("vc.py: msvs_find_valid_batch_script(): host_platform %s, target_platform %s req_target_platform:%s" % platforms) - host_platform, target_platform, req_target_platform) = platforms + host_platform, target_platform, req_target_platform = platforms try_target_archs = [target_platform] # VS2012 has a "cross compile" environment to build 64 bit -- cgit v0.12