From 7082c53ff6bc7454e4cd306123b2de8516a56c2d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:25:52 -0400 Subject: Fix #4312 and Windows on ARM64 support. Changes for #4312: * The installed VCS list is cached but has an indirect dependency on the value of the environment's TARGET_ARCH during construction. For the initial construction, force the env['TARGET_ARCH'] to be undefined and then restore the value after construction. * Protect against an empty regular expression list when iteratively removing default tool sequences. Changes for ARM64 host support: * Add configuration data structures to support ARM64 hosts and ARM64 native tools for VS2022. Update the tests for the renamed data structures. * Evaluate the PROCESSOR_ARCHITECTURE value from the windows registry before evaluating the os environment values for host architecture determination on Windows. * Set VSCMD_SKIP_SENDTELEMETRY=1 for arm32 process on arm64 host if not already defined to prevent powershell dll not found error. * The os environment variable value for PROCESSOR_ARCHITECTURE is AMD64 for Windows ARM64 hosts when using a python built for AMD64. Related MSVC changes for #4312 and ARM64 hosts: * Check that cl.exe exists in find_batch_file for VS6 to VS2015. Move the sdk batch file query code to its own function. Query for the sdk batch file only when all of the vc script host/target combinations have been exhausted and a valid script has not been found. * Hoist the query for the vc product directory outside the current vc script host/target loop. Catch the internal exceptions as before. * Clear the sdk batch file path for VCForPython as the sdk batch files do not appear to be updated during installation and do not point to the VCForPython installation location. * Move the sdk batch file determination to its own function. Defer evaluation of candidate sdk batch files until after all host/target combinations of the vc scripts have been evaluated. * Always check that cl.exe is found in vc script environment path. Miscellaneous: * Reorder and group exception definitions in Tool\MSCommon\vc.py by external exceptions and internal exceptions. * Adjust debug messages. * Convert the floating point msvc version to an integer version number for product range comparisons (e.g., 14.3 is 143). Adjust the comparison ranges accordingly. --- SCons/Platform/win32.py | 20 +- SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py | 29 +- SCons/Tool/MSCommon/common.py | 36 ++- SCons/Tool/MSCommon/vc.py | 429 ++++++++++++++++++++++------ SCons/Tool/MSCommon/vcTests.py | 48 +++- 5 files changed, 440 insertions(+), 122 deletions(-) diff --git a/SCons/Platform/win32.py b/SCons/Platform/win32.py index 990794f..e815913 100644 --- a/SCons/Platform/win32.py +++ b/SCons/Platform/win32.py @@ -299,6 +299,11 @@ SupportedArchitectureList = [ ), ArchDefinition( + 'arm64', + ['ARM64', 'aarch64', 'AARCH64', 'AArch64'], + ), + + ArchDefinition( 'ia64', ['IA64'], ), @@ -315,10 +320,21 @@ def get_architecture(arch=None): """Returns the definition for the specified architecture string. If no string is specified, the system default is returned (as defined - by the PROCESSOR_ARCHITEW6432 or PROCESSOR_ARCHITECTURE environment - variables). + by the registry PROCESSOR_ARCHITECTURE value, PROCESSOR_ARCHITEW6432 + environment variable, PROCESSOR_ARCHITECTURE environment variable, or + the platform machine). """ if arch is None: + if SCons.Util.can_read_reg: + try: + k=SCons.Util.RegOpenKeyEx(SCons.Util.hkey_mod.HKEY_LOCAL_MACHINE, + 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment') + val, tok = SCons.Util.RegQueryValueEx(k, 'PROCESSOR_ARCHITECTURE') + except SCons.Util.RegError: + val = '' + if val and val in SupportedArchitectureMap: + arch = val + if arch is None: arch = os.environ.get('PROCESSOR_ARCHITEW6432') if not arch: arch = os.environ.get('PROCESSOR_ARCHITECTURE') diff --git a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py index e1c05bc..7d0f0e4 100644 --- a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py +++ b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py @@ -188,19 +188,22 @@ def register_iserror(env, tool, msvc_exists_func): debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) # iteratively remove default tool sequences (longest to shortest) - re_nchar_min, re_tools_min = _Data.default_tools_re_list[-1] - if tools_nchar >= re_nchar_min and re_tools_min.search(tools): - # minimum characters satisfied and minimum pattern exists - for re_nchar, re_default_tool in _Data.default_tools_re_list: - if tools_nchar < re_nchar: - # not enough characters for pattern - continue - tools = re_default_tool.sub('', tools).strip(_Data.separator) - tools_nchar = len(tools) - debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) - if tools_nchar < re_nchar_min or not re_tools_min.search(tools): - # less than minimum characters or minimum pattern does not exist - break + if not _Data.default_tools_re_list: + debug('default_tools_re_list=%s', _Data.default_tools_re_list) + else: + re_nchar_min, re_tools_min = _Data.default_tools_re_list[-1] + if tools_nchar >= re_nchar_min and re_tools_min.search(tools): + # minimum characters satisfied and minimum pattern exists + for re_nchar, re_default_tool in _Data.default_tools_re_list: + if tools_nchar < re_nchar: + # not enough characters for pattern + continue + tools = re_default_tool.sub('', tools).strip(_Data.separator) + tools_nchar = len(tools) + debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) + if tools_nchar < re_nchar_min or not re_tools_min.search(tools): + # less than minimum characters or minimum pattern does not exist + break # construct non-default list(s) tools set tools_set = {msvc_tool for msvc_tool in tools.split(_Data.separator) if msvc_tool} diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index ad4c827..df8e6c8 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -209,6 +209,17 @@ def has_reg(value): # Functions for fetching environment variable settings from batch files. +def _force_vscmd_skip_sendtelemetry(env): + + if 'VSCMD_SKIP_SENDTELEMETRY' in env['ENV']: + return False + + env['ENV']['VSCMD_SKIP_SENDTELEMETRY'] = '1' + debug("force env['ENV']['VSCMD_SKIP_SENDTELEMETRY']=%s", env['ENV']['VSCMD_SKIP_SENDTELEMETRY']) + + return True + + def normalize_env(env, keys, force=False): """Given a dictionary representing a shell environment, add the variables from os.environ needed for the processing of .bat files; the keys are @@ -257,7 +268,7 @@ def normalize_env(env, keys, force=False): return normenv -def get_output(vcbat, args=None, env=None): +def get_output(vcbat, args=None, env=None, skip_sendtelemetry=False): """Parse the output of given bat file, with given args.""" if env is None: @@ -296,20 +307,23 @@ def get_output(vcbat, args=None, env=None): ] env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False) + if skip_sendtelemetry: + _force_vscmd_skip_sendtelemetry(env) + if args: debug("Calling '%s %s'", vcbat, args) - popen = SCons.Action._subproc(env, - '"%s" %s & set' % (vcbat, args), - stdin='devnull', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + cmd_str = '"%s" %s & set' % (vcbat, args) else: debug("Calling '%s'", vcbat) - popen = SCons.Action._subproc(env, - '"%s" & set' % vcbat, - stdin='devnull', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + cmd_str = '"%s" & set' % vcbat + + popen = SCons.Action._subproc( + env, + cmd_str, + stdin='devnull', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) # Use the .stdout and .stderr attributes directly because the # .communicate() method uses the threading module on Windows diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 7871940..64b4cde 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -43,6 +43,7 @@ import SCons.compat import subprocess import os import platform +import sysconfig from pathlib import Path from string import digits as string_digits from subprocess import PIPE @@ -69,8 +70,7 @@ from .MSVC.Exceptions import ( MSVCToolsetVersionNotFound, ) -class UnsupportedVersion(VisualCException): - pass +# external exceptions class MSVCUnsupportedHostArch(VisualCException): pass @@ -78,21 +78,35 @@ class MSVCUnsupportedHostArch(VisualCException): class MSVCUnsupportedTargetArch(VisualCException): pass -class MissingConfiguration(VisualCException): +class MSVCScriptNotFound(MSVCUserError): pass -class NoVersionFound(VisualCException): +class MSVCUseSettingsError(MSVCUserError): pass -class BatchFileExecutionError(VisualCException): +# internal exceptions + +class UnsupportedVersion(VisualCException): pass -class MSVCScriptNotFound(MSVCUserError): +class MissingConfiguration(VisualCException): pass -class MSVCUseSettingsError(MSVCUserError): +class BatchFileExecutionError(VisualCException): pass +# undefined object for dict.get() in case key exists and value is None +UNDEFINED = object() + +# powershell error sending telemetry for arm32 process on arm64 host (VS2019+): +# True: force VSCMD_SKIP_SENDTELEMETRY=1 (if necessary) +# False: do nothing +_ARM32_ON_ARM64_SKIP_SENDTELEMETRY = True + +# MSVC 9.0 preferred query order: +# True: VCForPython, VisualStudio +# FAlse: VisualStudio, VCForPython +_VC90_Prefer_VCForPython = True # Dict to 'canonalize' the arch _ARCH_TO_CANONICAL = { @@ -282,7 +296,9 @@ def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host # The cl path fragment under the toolset version folder is the second value of # the stored tuple. -_GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { +# 14.3 (VS2022) and later + +_GE2022_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { ('amd64', 'amd64') : ('vcvars64.bat', ('bin', 'Hostx64', 'x64')), ('amd64', 'x86') : ('vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86')), @@ -294,11 +310,66 @@ _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { ('x86', 'arm') : ('vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm')), ('x86', 'arm64') : ('vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64')), + ('arm64', 'amd64') : ('vcvarsarm64_amd64.bat', ('bin', 'Hostarm64', 'arm64_amd64')), + ('arm64', 'x86') : ('vcvarsarm64_x86.bat', ('bin', 'Hostarm64', 'arm64_x86')), + ('arm64', 'arm') : ('vcvarsarm64_arm.bat', ('bin', 'Hostarm64', 'arm64_arm')), + ('arm64', 'arm64') : ('vcvarsarm64.bat', ('bin', 'Hostarm64', 'arm64')), + } -_GE2017_HOST_TARGET_CFG = _host_target_config_factory( +_GE2022_HOST_TARGET_CFG = _host_target_config_factory( - label = 'GE2017', + label = 'GE2022', + + host_all_hosts = OrderedDict([ + ('amd64', ['amd64', 'x86']), + ('x86', ['x86']), + ('arm64', ['arm64', 'amd64', 'x86']), + ('arm', ['x86']), + ]), + + host_all_targets = { + 'amd64': ['amd64', 'x86', 'arm64', 'arm'], + 'x86': ['x86', 'amd64', 'arm', 'arm64'], + 'arm64': ['arm64', 'amd64', 'arm', 'x86'], + 'arm': [], + }, + + host_def_targets = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm64': ['arm64', 'amd64', 'arm', 'x86'], + 'arm': ['arm'], + }, + +) + +# debug("_GE2022_HOST_TARGET_CFG: %s", _GE2022_HOST_TARGET_CFG) + +# 14.2 (VS2019) to 14.1 (VS2017) + +_LE2019_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { + + ('amd64', 'amd64') : ('vcvars64.bat', ('bin', 'Hostx64', 'x64')), + ('amd64', 'x86') : ('vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86')), + ('amd64', 'arm') : ('vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm')), + ('amd64', 'arm64') : ('vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64')), + + ('x86', 'amd64') : ('vcvarsx86_amd64.bat', ('bin', 'Hostx86', 'x64')), + ('x86', 'x86') : ('vcvars32.bat', ('bin', 'Hostx86', 'x86')), + ('x86', 'arm') : ('vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm')), + ('x86', 'arm64') : ('vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64')), + + ('arm64', 'amd64') : ('vcvars64.bat', ('bin', 'Hostx64', 'x64')), + ('arm64', 'x86') : ('vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86')), + ('arm64', 'arm') : ('vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm')), + ('arm64', 'arm64') : ('vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64')), + +} + +_LE2019_HOST_TARGET_CFG = _host_target_config_factory( + + label = 'LE2019', host_all_hosts = OrderedDict([ ('amd64', ['amd64', 'x86']), @@ -310,20 +381,20 @@ _GE2017_HOST_TARGET_CFG = _host_target_config_factory( host_all_targets = { 'amd64': ['amd64', 'x86', 'arm64', 'arm'], 'x86': ['x86', 'amd64', 'arm', 'arm64'], - 'arm64': [], + 'arm64': ['arm64', 'amd64', 'arm', 'x86'], 'arm': [], }, host_def_targets = { 'amd64': ['amd64', 'x86'], 'x86': ['x86'], - 'arm64': ['arm64', 'arm'], + 'arm64': ['arm64', 'amd64', 'arm', 'x86'], 'arm': ['arm'], }, ) -# debug("_GE2017_HOST_TARGET_CFG: %s", _GE2017_HOST_TARGET_CFG) +# debug("_LE2019_HOST_TARGET_CFG: %s", _LE2019_HOST_TARGET_CFG) # 14.0 (VS2015) to 8.0 (VS2005) @@ -345,6 +416,10 @@ _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS = { ('x86', 'arm') : ('x86_arm', ('bin', 'x86_arm')), ('x86', 'ia64') : ('x86_ia64', ('bin', 'x86_ia64')), + ('arm64', 'amd64') : ('amd64', ('bin', 'amd64')), + ('arm64', 'x86') : ('amd64_x86', ('bin', 'amd64_x86')), + ('arm64', 'arm') : ('amd64_arm', ('bin', 'amd64_arm')), + ('arm', 'arm') : ('arm', ('bin', 'arm')), ('ia64', 'ia64') : ('ia64', ('bin', 'ia64')), @@ -357,6 +432,7 @@ _LE2015_HOST_TARGET_CFG = _host_target_config_factory( host_all_hosts = OrderedDict([ ('amd64', ['amd64', 'x86']), ('x86', ['x86']), + ('arm64', ['amd64', 'x86']), ('arm', ['arm']), ('ia64', ['ia64']), ]), @@ -364,6 +440,7 @@ _LE2015_HOST_TARGET_CFG = _host_target_config_factory( host_all_targets = { 'amd64': ['amd64', 'x86', 'arm'], 'x86': ['x86', 'amd64', 'arm', 'ia64'], + 'arm64': ['amd64', 'x86', 'arm'], 'arm': ['arm'], 'ia64': ['ia64'], }, @@ -371,6 +448,7 @@ _LE2015_HOST_TARGET_CFG = _host_target_config_factory( host_def_targets = { 'amd64': ['amd64', 'x86'], 'x86': ['x86'], + 'arm64': ['amd64', 'arm', 'x86'], 'arm': ['arm'], 'ia64': ['ia64'], }, @@ -391,16 +469,19 @@ _LE2003_HOST_TARGET_CFG = _host_target_config_factory( host_all_hosts = OrderedDict([ ('amd64', ['x86']), ('x86', ['x86']), + ('arm64', ['x86']), ]), host_all_targets = { 'amd64': ['x86'], 'x86': ['x86'], + 'arm64': ['x86'], }, host_def_targets = { 'amd64': ['x86'], 'x86': ['x86'], + 'arm64': ['x86'], }, ) @@ -444,28 +525,54 @@ def get_host_platform(host_platform): return host +_native_host_architecture = None + +def get_native_host_architecture(): + """Return the native host architecture.""" + global _native_host_architecture + + if _native_host_architecture is None: + + try: + arch = common.read_reg( + r'SYSTEM\CurrentControlSet\Control\Session Manager\Environment\PROCESSOR_ARCHITECTURE' + ) + except OSError: + arch = None + + if not arch: + arch = platform.machine() + + _native_host_architecture = arch + + return _native_host_architecture + _native_host_platform = None def get_native_host_platform(): global _native_host_platform if _native_host_platform is None: - - _native_host_platform = get_host_platform(platform.machine()) + arch = get_native_host_architecture() + _native_host_platform = get_host_platform(arch) return _native_host_platform def get_host_target(env, msvc_version, all_host_targets=False): vernum = float(get_msvc_version_numeric(msvc_version)) - - if vernum > 14: - # 14.1 (VS2017) and later - host_target_cfg = _GE2017_HOST_TARGET_CFG - elif 14 >= vernum >= 8: + vernum_int = int(vernum * 10) + + if vernum_int >= 143: + # 14.3 (VS2022) and later + host_target_cfg = _GE2022_HOST_TARGET_CFG + elif 143 > vernum_int >= 141: + # 14.2 (VS2019) to 14.1 (VS2017) + host_target_cfg = _LE2019_HOST_TARGET_CFG + elif 141 > vernum_int >= 80: # 14.0 (VS2015) to 8.0 (VS2005) host_target_cfg = _LE2015_HOST_TARGET_CFG - else: + else: # 80 > vernum_int # 7.1 (VS2003) and earlier host_target_cfg = _LE2003_HOST_TARGET_CFG @@ -520,6 +627,53 @@ def get_host_target(env, msvc_version, all_host_targets=False): return host_platform, target_platform, host_target_list +_arm32_process_arm64_host = None + +def is_arm32_process_arm64_host(): + global _arm32_process_arm64_host + + if _arm32_process_arm64_host is None: + + host = get_native_host_architecture() + host = _ARCH_TO_CANONICAL.get(host.lower(),'') + host_isarm64 = host == 'arm64' + + process = sysconfig.get_platform() + process_isarm32 = process == 'win-arm32' + + _arm32_process_arm64_host = host_isarm64 and process_isarm32 + + return _arm32_process_arm64_host + +_check_skip_sendtelemetry = None + +def _skip_sendtelemetry(env): + global _check_skip_sendtelemetry + + if _check_skip_sendtelemetry is None: + + if _ARM32_ON_ARM64_SKIP_SENDTELEMETRY and is_arm32_process_arm64_host(): + _check_skip_sendtelemetry = True + else: + _check_skip_sendtelemetry = False + + if not _check_skip_sendtelemetry: + return False + + msvc_version = env.get('MSVC_VERSION') if env else None + if not msvc_version: + msvc_version = msvc_default_version(env) + + if not msvc_version: + return False + + vernum = float(get_msvc_version_numeric(msvc_version)) + if vernum < 14.2: # VS2019 + return False + + # arm32 process, arm64 host, VS2019+ + return True + # If you update this, update SupportedVSList in Tool/MSCommon/vs.py, and the # MSVC_VERSION documentation in Tool/msvc.xml. _VCVER = [ @@ -589,6 +743,9 @@ _VCVER_TO_PRODUCT_DIR = { '9.0': [ (SCons.Util.HKEY_CURRENT_USER, r'Microsoft\DevDiv\VCForPython\9.0\installdir',), (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VisualStudio\9.0\Setup\VC\ProductDir',), + ] if _VC90_Prefer_VCForPython else [ + (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VisualStudio\9.0\Setup\VC\ProductDir',), + (SCons.Util.HKEY_CURRENT_USER, r'Microsoft\DevDiv\VCForPython\9.0\installdir',), ], '9.0Exp': [ (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VCExpress\9.0\Setup\VC\ProductDir'), @@ -763,7 +920,7 @@ def find_vc_pdir(env, msvc_version): raise MissingConfiguration("registry dir {} not found on the filesystem".format(comps)) return None -def find_batch_file(env, msvc_version, host_arch, target_arch): +def find_batch_file(msvc_version, host_arch, target_arch, pdir): """ Find the location of the batch script which should set up the compiler for any TARGET_ARCH whose compilers were installed by Visual Studio/VCExpress @@ -772,52 +929,73 @@ def find_batch_file(env, msvc_version, host_arch, target_arch): scripts named with a host_target pair that calls vcvarsall.bat properly, so use that and return an empty argument. """ - pdir = find_vc_pdir(env, msvc_version) - if pdir is None: - raise NoVersionFound("No version of Visual Studio found") - debug('looking in %s', pdir) # filter out e.g. "Exp" from the version name - msvc_ver_numeric = get_msvc_version_numeric(msvc_version) - vernum = float(msvc_ver_numeric) + vernum = float(get_msvc_version_numeric(msvc_version)) + vernum_int = int(vernum * 10) + + sdk_pdir = pdir arg = '' vcdir = None + clexe = None - if vernum > 14: - # 14.1 (VS2017) and later + if vernum_int >= 143: + # 14.3 (VS2022) and later + batfiledir = os.path.join(pdir, "Auxiliary", "Build") + batfile, _ = _GE2022_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host_arch, target_arch)] + batfilename = os.path.join(batfiledir, batfile) + vcdir = pdir + elif 143 > vernum_int >= 141: + # 14.2 (VS2019) to 14.1 (VS2017) batfiledir = os.path.join(pdir, "Auxiliary", "Build") - batfile, _ = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host_arch, target_arch)] + batfile, _ = _LE2019_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host_arch, target_arch)] batfilename = os.path.join(batfiledir, batfile) vcdir = pdir - elif 14 >= vernum >= 8: + elif 141 > vernum_int >= 80: # 14.0 (VS2015) to 8.0 (VS2005) - arg, _ = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host_arch, target_arch)] + arg, cl_path_comps = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host_arch, target_arch)] batfilename = os.path.join(pdir, "vcvarsall.bat") if msvc_version == '9.0' and not os.path.exists(batfilename): # Visual C++ for Python batch file is in installdir (root) not productdir (vc) batfilename = os.path.normpath(os.path.join(pdir, os.pardir, "vcvarsall.bat")) - else: + # Visual C++ for Python sdk batch files do not point to the VCForPython installation + sdk_pdir = None + clexe = os.path.join(pdir, *cl_path_comps, _CL_EXE_NAME) + else: # 80 > vernum_int # 7.1 (VS2003) and earlier pdir = os.path.join(pdir, "Bin") batfilename = os.path.join(pdir, "vcvars32.bat") + clexe = os.path.join(pdir, _CL_EXE_NAME) if not os.path.exists(batfilename): - debug("Not found: %s", batfilename) + debug("batch file not found: %s", batfilename) batfilename = None + if clexe and not os.path.exists(clexe): + debug("cl.exe not found: %s", clexe) + batfilename = None + + return batfilename, arg, vcdir, sdk_pdir + +def find_batch_file_sdk(host_arch, target_arch, sdk_pdir): + """ + Find the location of the sdk batch script which should set up the compiler + for any TARGET_ARCH whose compilers were installed by Visual Studio/VCExpress + """ + installed_sdks = get_installed_sdks() for _sdk in installed_sdks: sdk_bat_file = _sdk.get_sdk_vc_script(host_arch, target_arch) if not sdk_bat_file: - debug("batch file not found:%s", _sdk) + debug("sdk batch file not found:%s", _sdk) else: - sdk_bat_file_path = os.path.join(pdir, sdk_bat_file) + sdk_bat_file_path = os.path.join(sdk_pdir, sdk_bat_file) if os.path.exists(sdk_bat_file_path): debug('sdk_bat_file_path:%s', sdk_bat_file_path) - return batfilename, arg, vcdir, sdk_bat_file_path + return sdk_bat_file_path - return batfilename, arg, vcdir, None + return None __INSTALLED_VCS_RUN = None _VC_TOOLS_VERSION_FILE_PATH = ['Auxiliary', 'Build', 'Microsoft.VCToolsVersion.default.txt'] @@ -852,9 +1030,10 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): host_platform, target_platform, host_target_list = platforms vernum = float(get_msvc_version_numeric(msvc_version)) + vernum_int = int(vernum * 10) # make sure the cl.exe exists meaning the tool is installed - if vernum > 14: + if vernum_int >= 141: # 14.1 (VS2017) and later # 2017 and newer allowed multiple versions of the VC toolset to be # installed at the same time. This changes the layout. @@ -871,11 +1050,18 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('failed to find MSVC version in %s', default_toolset_file) return False + if vernum_int >= 143: + # 14.3 (VS2022) and later + host_target_batchfile_clpathcomps = _GE2022_HOST_TARGET_BATCHFILE_CLPATHCOMPS + else: + # 14.2 (VS2019) to 14.1 (VS2017) + host_target_batchfile_clpathcomps = _LE2019_HOST_TARGET_BATCHFILE_CLPATHCOMPS + for host_platform, target_platform in host_target_list: debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - batchfile_clpathcomps = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS.get((host_platform, target_platform), None) + batchfile_clpathcomps = host_target_batchfile_clpathcomps.get((host_platform, target_platform), None) if batchfile_clpathcomps is None: debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) continue @@ -888,7 +1074,7 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('found %s!', _CL_EXE_NAME) return True - elif 14 >= vernum >= 8: + elif 141 > vernum_int >= 80: # 14.0 (VS2015) to 8.0 (VS2005) for host_platform, target_platform in host_target_list: @@ -908,7 +1094,7 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('found %s', _CL_EXE_NAME) return True - elif 8 > vernum >= 6: + elif 80 > vernum_int >= 60: # 7.1 (VS2003) to 6.0 (VS6) # quick check for vc_dir/bin and vc_dir/ before walk @@ -928,7 +1114,7 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): return False else: - # version not support return false + # version not supported return false debug('unsupported MSVC version: %s', str(vernum)) return False @@ -939,6 +1125,13 @@ def get_installed_vcs(env=None): if __INSTALLED_VCS_RUN is not None: return __INSTALLED_VCS_RUN + save_target_arch = env.get('TARGET_ARCH', UNDEFINED) if env else None + force_target = env and save_target_arch and save_target_arch != UNDEFINED + + if force_target: + del env['TARGET_ARCH'] + debug("delete env['TARGET_ARCH']") + installed_versions = [] for ver in _VCVER: @@ -960,7 +1153,12 @@ def get_installed_vcs(env=None): except VisualCException as e: debug('did not find VC %s: caught exception %s', ver, str(e)) + if force_target: + env['TARGET_ARCH'] = save_target_arch + debug("restore env['TARGET_ARCH']=%s", save_target_arch) + __INSTALLED_VCS_RUN = installed_versions + debug("__INSTALLED_VCS_RUN=%s", __INSTALLED_VCS_RUN) return __INSTALLED_VCS_RUN def reset_installed_vcs(): @@ -982,6 +1180,19 @@ def get_installed_vcs_components(env=None): msvc_version_component_defs = [MSVC.Util.msvc_version_components(vcver) for vcver in vcs] return msvc_version_component_defs +def _check_cl_exists_in_script_env(data): + """Find cl.exe in the script environment path.""" + cl_path = None + if data and 'PATH' in data: + for p in data['PATH']: + cl_exe = os.path.join(p, _CL_EXE_NAME) + if os.path.exists(cl_exe): + cl_path = cl_exe + break + have_cl = True if cl_path else False + debug('have_cl: %s, cl_path: %s', have_cl, cl_path) + return have_cl, cl_path + # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" # in multiple environments, for example: @@ -1027,7 +1238,8 @@ def script_env(env, script, args=None): cache_data = None if cache_data is None: - stdout = common.get_output(script, args) + skip_sendtelemetry = _skip_sendtelemetry(env) + stdout = common.get_output(script, args, skip_sendtelemetry=skip_sendtelemetry) cache_data = common.parse_output(stdout) # debug(stdout) @@ -1044,12 +1256,7 @@ def script_env(env, script, args=None): if script_errlog: script_errmsg = '\n'.join(script_errlog) - have_cl = False - if cache_data and 'PATH' in cache_data: - for p in cache_data['PATH']: - if os.path.exists(os.path.join(p, _CL_EXE_NAME)): - have_cl = True - break + have_cl, _ = _check_cl_exists_in_script_env(cache_data) debug( 'script=%s args=%s have_cl=%s, errors=%s', @@ -1130,6 +1337,18 @@ def msvc_find_valid_batch_script(env, version): get it right. """ + # Find the product directory + pdir = None + try: + pdir = find_vc_pdir(env, version) + except UnsupportedVersion: + # Unsupported msvc version (raise MSVCArgumentError?) + pass + except MissingConfiguration: + # Found version, directory missing + pass + debug('product directory: version=%s, pdir=%s', version, pdir) + # Find the host, target, and all candidate (host, target) platform combinations: platforms = get_host_target(env, version) debug("host_platform %s, target_platform %s host_target_list %s", *platforms) @@ -1137,48 +1356,81 @@ def msvc_find_valid_batch_script(env, version): d = None version_installed = False - for host_arch, target_arch, in host_target_list: - # Set to current arch. - env['TARGET_ARCH'] = target_arch - arg = '' - # Try to locate a batch file for this host/target platform combo - try: - (vc_script, arg, vc_dir, sdk_script) = find_batch_file(env, version, host_arch, target_arch) - debug('vc_script:%s vc_script_arg:%s sdk_script:%s', vc_script, arg, sdk_script) - version_installed = True - except VisualCException as e: - msg = str(e) - debug('Caught exception while looking for batch file (%s)', msg) - version_installed = False - continue + if pdir: + + # Query all candidate sdk (host, target, sdk_pdir) after vc_script pass if necessary + sdk_queries = [] + + for host_arch, target_arch, in host_target_list: + # Set to current arch. + env['TARGET_ARCH'] = target_arch + arg = '' + + # Try to locate a batch file for this host/target platform combo + try: + (vc_script, arg, vc_dir, sdk_pdir) = find_batch_file(version, host_arch, target_arch, pdir) + debug('vc_script:%s vc_script_arg:%s', vc_script, arg) + version_installed = True + except VisualCException as e: + msg = str(e) + debug('Caught exception while looking for batch file (%s)', msg) + version_installed = False + continue + + # Save (host, target, sdk_pdir) platform combo for sdk queries + if sdk_pdir: + sdk_query = (host_arch, target_arch, sdk_pdir) + if sdk_query not in sdk_queries: + debug('save sdk_query host=%s, target=%s, sdk_pdir=%s', host_arch, target_arch, sdk_pdir) + sdk_queries.append(sdk_query) + + if not vc_script: + continue - # Try to use the located batch file for this host/target platform combo - debug('use_script 2 %s, args:%s', repr(vc_script), arg) - found = None - if vc_script: + # Try to use the located batch file for this host/target platform combo arg = MSVC.ScriptArguments.msvc_script_arguments(env, version, vc_dir, arg) + debug('trying vc_script:%s, vc_script_args:%s', repr(vc_script), arg) try: d = script_env(env, vc_script, args=arg) - found = vc_script except BatchFileExecutionError as e: - debug('use_script 3: failed running VC script %s: %s: Error:%s', repr(vc_script), arg, e) - vc_script=None + debug('failed vc_script:%s, vc_script_args:%s, error:%s', repr(vc_script), arg, e) + vc_script = None continue - if not vc_script and sdk_script: - debug('use_script 4: trying sdk script: %s', sdk_script) - try: - d = script_env(env, sdk_script) - found = sdk_script - except BatchFileExecutionError as e: - debug('use_script 5: failed running SDK script %s: Error:%s', repr(sdk_script), e) + + have_cl, _ = _check_cl_exists_in_script_env(d) + if not have_cl: + debug('skip cl.exe not found vc_script:%s, vc_script_args:%s', repr(vc_script), arg) continue - elif not vc_script and not sdk_script: - debug('use_script 6: Neither VC script nor SDK script found') - continue - debug("Found a working script/target: %s/%s", repr(found), arg) - break # We've found a working target_platform, so stop looking + debug("Found a working script/target: %s/%s", repr(vc_script), arg) + break # We've found a working target_platform, so stop looking + + if not d: + for host_arch, target_arch, sdk_pdir in sdk_queries: + # Set to current arch. + env['TARGET_ARCH'] = target_arch + + sdk_script = find_batch_file_sdk(host_arch, target_arch, sdk_pdir) + if not sdk_script: + continue + + # Try to use the sdk batch file for this (host, target, sdk_pdir) combo + debug('trying sdk_script:%s', repr(sdk_script)) + try: + d = script_env(env, sdk_script) + version_installed = True + except BatchFileExecutionError as e: + debug('failed sdk_script:%s, error=%s', repr(sdk_script), e) + continue + + have_cl, _ = _check_cl_exists_in_script_env(d) + if not have_cl: + debug('skip cl.exe not found sdk_script:%s', repr(sdk_script)) + continue + + debug("Found a working script/target: %s", repr(sdk_script)) + break # We've found a working script, so stop looking # If we cannot find a viable installed compiler, reset the TARGET_ARCH # To it's initial value @@ -1206,8 +1458,6 @@ def msvc_find_valid_batch_script(env, version): return d -_UNDEFINED = object() - def get_use_script_use_settings(env): # use_script use_settings return values action @@ -1217,9 +1467,9 @@ def get_use_script_use_settings(env): # None (documentation) or evaluates False (code): bypass detection # need to distinguish between undefined and None - use_script = env.get('MSVC_USE_SCRIPT', _UNDEFINED) + use_script = env.get('MSVC_USE_SCRIPT', UNDEFINED) - if use_script != _UNDEFINED: + if use_script != UNDEFINED: # use_script defined, use_settings ignored (not type checked) return use_script, None @@ -1293,8 +1543,7 @@ def msvc_exists(env=None, version=None): rval = len(vcs) > 0 else: rval = version in vcs - if not rval: - debug('version=%s, return=%s', repr(version), rval) + debug('version=%s, return=%s', repr(version), rval) return rval def msvc_setup_env_user(env=None): diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index 3e37def..a8b139a 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -132,12 +132,48 @@ class MSVcTestCase(unittest.TestCase): print("Failed trying to write :%s :%s" % (tools_version_file, e)) - # Now walk all the valid combinations of host/target for 14.1 (VS2017) and later - vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_CFG.all_pairs + # Test 14.3 (VS2022) and later + vc_ge2022_list = SCons.Tool.MSCommon.vc._GE2022_HOST_TARGET_CFG.all_pairs - for host, target in vc_ge2017_list: - batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] - # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) + for host, target in vc_ge2022_list: + batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2022_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] + # print("GE 14.3 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) + + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.', 'Tools', 'MSVC', MS_TOOLS_VERSION, *clpathcomps) + MSVcTestCase._createDummyCl(path, add_bin=False) + result=check(env, '.', '14.3') + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) + + # Now test bogus value for HOST_ARCH + env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} + try: + result=check(env, '.', '14.3') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedHostArch: + pass + else: + self.fail('Did not fail when HOST_ARCH specified as: %s' % env['HOST_ARCH']) + + # Now test bogus value for TARGET_ARCH + env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} + try: + result=check(env, '.', '14.3') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedTargetArch: + pass + else: + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) + + # Test 14.2 (VS2019) to 14.1 (VS2017) versions + vc_le2019_list = SCons.Tool.MSCommon.vc._LE2019_HOST_TARGET_CFG.all_pairs + + for host, target in vc_le2019_list: + batfile, clpathcomps = SCons.Tool.MSCommon.vc._LE2019_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] + # print("LE 14.2 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) env={'TARGET_ARCH':target, 'HOST_ARCH':host} path = os.path.join('.', 'Tools', 'MSVC', MS_TOOLS_VERSION, *clpathcomps) @@ -173,7 +209,7 @@ class MSVcTestCase(unittest.TestCase): for host, target in vc_le2015_list: batarg, clpathcomps = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host, target)] - # print("LE 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpathcomps)) + # print("LE 14.0 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpathcomps)) env={'TARGET_ARCH':target, 'HOST_ARCH':host} path = os.path.join('.', *clpathcomps) MSVcTestCase._createDummyCl(path, add_bin=False) -- cgit v0.12