From 735147d4d6434ba236571be3ec88f2ec53ea6554 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 4 Mar 2020 10:37:54 -0700 Subject: Pass some env vars that affect msvs tool setup. MSCommon already makes sure a few variables are propagated before calling the the dev env setup script (vcvarsall.bat or relative). This change adds two more: VSCMD_DEBUG enables VsDevCmd.bat (which is eventually called) to log the environment before and after setup, and VSCMD_SKIP_SENDTELEMETRY, if set, inhibits the new (VS 2019) functionality to send telemetry information. The change is motivated by the telemetry code generating undecodable output for certain locales. This should allow suppressing it, but doesn't really fix the underlying problem that we're not handling text right on Windows (on Py2 this would not have failed, but that was an error as well - it should have). Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 2 ++ src/engine/SCons/Tool/MSCommon/common.py | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 514ecf7..958a475 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -84,6 +84,8 @@ From Rob Boehne - Fixed bug where changing TEXTFILESUFFIX would cause Substfile() to rebuild. (Github Issue #3540) - Script/Main.py now uses importlib instead of imp module. - Drop some Python 2-isms. + - Pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc tool setup + if set in environment. RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index 7f832c6..422d897 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -185,16 +185,21 @@ def get_output(vcbat, args=None, env=None): # Create a blank environment, for use in launching the tools env = SCons.Environment.Environment(tools=[]) - # TODO: This is a hard-coded list of the variables that (may) need - # to be imported from os.environ[] for v[sc]*vars*.bat file - # execution to work. This list should really be either directly - # controlled by vc.py, or else derived from the common_tools_var - # settings in vs.py. + # TODO: Hard-coded list of the variables that (may) need to be + # imported from os.environ[] for the chain of development batch + # files to execute correctly. One call to vcvars*.bat may + # end up running a dozen or more scripts, changes not only with + # each release but with what is installed at the time. We think + # in modern installations most are set along the way and don't + # need to be picked from the env, but include these for safety's sake. + # Any VSCMD variables definitely are picked from the env and + # control execution in interesting ways. + # Note these really should be unified - either controlled by vs.py, + # or synced with the the common_tools_var # settings in vs.py. vs_vc_vars = [ - 'COMSPEC', - # VS100 and VS110: Still set, but modern MSVC setup scripts will - # discard these if registry has values. However Intel compiler setup - # script still requires these as of 2013/2014. + 'COMSPEC', # path to "shell" + 'VS160COMNTOOLS', # path to common tools for given version + 'VS150COMNTOOLS', 'VS140COMNTOOLS', 'VS120COMNTOOLS', 'VS110COMNTOOLS', @@ -204,6 +209,8 @@ def get_output(vcbat, args=None, env=None): 'VS71COMNTOOLS', 'VS70COMNTOOLS', 'VS60COMNTOOLS', + 'VSCMD_DEBUG', # enable logging and other debug aids + 'VSCMD_SKIP_SENDTELEMETRY', ] env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False) @@ -237,7 +244,7 @@ def get_output(vcbat, args=None, env=None): if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr) + sys.stderr.write(stderr.decode("mbcs")) if popen.wait() != 0: raise IOError(stderr.decode("mbcs")) -- cgit v0.12 From b18601eb0e057afcbdbc92ff516742767361f192 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 Mar 2020 08:22:08 -0600 Subject: [PR #3576] add powershell path We pass a very minimal PATH to the call to vcvars*. Apparently there are circumstances where not having Powershell in it can cause failures (relates to the same telemetry call that is the subject of this PR). Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 5 +++-- src/engine/SCons/Tool/MSCommon/common.py | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 958a475..2147f49 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -84,8 +84,9 @@ From Rob Boehne - Fixed bug where changing TEXTFILESUFFIX would cause Substfile() to rebuild. (Github Issue #3540) - Script/Main.py now uses importlib instead of imp module. - Drop some Python 2-isms. - - Pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc tool setup - if set in environment. + - MSVC updates: pass on VSCMD_DEBUG and VSCMD_SKIP_SENDTELEMETRY to msvc + tool setup if set in environment. Add powershell to default env + (used to call telemetry script). RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index 422d897..c066fd8 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -156,15 +156,14 @@ def normalize_env(env, keys, force=False): if k in os.environ and (force or k not in normenv): normenv[k] = os.environ[k] - # This shouldn't be necessary, since the default environment should include system32, - # but keep this here to be safe, since it's needed to find reg.exe which the MSVC - # bat scripts use. - sys32_dir = os.path.join(os.environ.get("SystemRoot", - os.environ.get("windir", r"C:\Windows\system32")), - "System32") - - if sys32_dir not in normenv['PATH']: - normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_dir + # add some things to PATH to prevent problems: + # Shouldn't be necessary to add system32, since the default environment + # should include it, but keep this here to be safe (needed for reg.exe) + sys32_dir = os.path.join( + os.environ.get("SystemRoot", os.environ.get("windir", r"C:\Windows")), "System32" +) + if sys32_dir not in normenv["PATH"]: + normenv["PATH"] = normenv["PATH"] + os.pathsep + sys32_dir # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized" # error starting with Visual Studio 2017, although the script still @@ -173,8 +172,14 @@ def normalize_env(env, keys, force=False): if sys32_wbem_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir - debug("PATH: %s" % normenv['PATH']) + # Without Powershell in PATH, an internal call to a telemetry + # function (starting with a VS2019 update) can fail + # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this. + sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\1.0') + if sys32_ps_dir not in normenv['PATH']: + normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir + debug("PATH: %s" % normenv['PATH']) return normenv -- cgit v0.12 From ddc4bd8a969853b27b8275fab81a6fafc9bff732 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 Mar 2020 09:59:52 -0600 Subject: [PR #3576] fix typo in powershell path Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index c066fd8..b7203a9 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -175,7 +175,7 @@ def normalize_env(env, keys, force=False): # Without Powershell in PATH, an internal call to a telemetry # function (starting with a VS2019 update) can fail # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this. - sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\1.0') + sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\v1.0') if sys32_ps_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir -- cgit v0.12 From 67aea476a01613edc6861775fbbc59b642604bbb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 19 Mar 2020 08:46:41 -0600 Subject: [PR #3576] change decoding to "oem" in vcvars* runner Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b7203a9..b90c25a 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -246,21 +246,28 @@ def get_output(vcbat, args=None, env=None): # debug('get_output():stdout:%s'%stdout) # debug('get_output():stderr:%s'%stderr) + # Ongoing problems getting non-corrupted text led to this + # changing to "oem" from "mbcs" - the scripts run presumably + # attached to a console, so some particular rules apply. if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr.decode("mbcs")) + sys.stderr.write(stderr.decode("oem")) if popen.wait() != 0: - raise IOError(stderr.decode("mbcs")) + raise IOError(stderr.decode("oem")) - output = stdout.decode("mbcs") - return output + return stdout.decode("oem") -KEEPLIST = ("INCLUDE", "LIB", "LIBPATH", "PATH", 'VSCMD_ARG_app_plat', - 'VCINSTALLDIR', # needed by clang -VS 2017 and newer - 'VCToolsInstallDir', # needed by clang - VS 2015 and older - ) +KEEPLIST = ( + "INCLUDE", + "LIB", + "LIBPATH", + "PATH", + "VSCMD_ARG_app_plat", + "VCINSTALLDIR", # needed by clang -VS 2017 and newer + "VCToolsInstallDir", # needed by clang - VS 2015 and older +) def parse_output(output, keep=KEEPLIST): -- cgit v0.12 From 433c62479cc022ae7b07cf46a8c772cedae2842f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 7 Apr 2020 10:04:24 -0600 Subject: [PR #3576] workaroud "oem" coding not on Py3.5 Use a Windows API to fetch the console output code page if Python < 3.6, where the encoding name "oem" is not yet defined (this was the original proposal in issue #3572) Signed-off-by: Mats Wichmann --- src/engine/SCons/Tool/MSCommon/common.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index b90c25a..b2b5de9 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -249,14 +249,21 @@ def get_output(vcbat, args=None, env=None): # Ongoing problems getting non-corrupted text led to this # changing to "oem" from "mbcs" - the scripts run presumably # attached to a console, so some particular rules apply. + # Unfortunately, "oem" not defined in Python 3.5, so get another way + if sys.version_info.major == 3 and sys.version_info.minor < 6: + from ctypes import windll + + OEM = "cp{}".format(windll.kernel32.GetConsoleOutputCP()) + else: + OEM = "oem" if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr.decode("oem")) + sys.stderr.write(stderr.decode(OEM)) if popen.wait() != 0: - raise IOError(stderr.decode("oem")) + raise IOError(stderr.decode(OEM)) - return stdout.decode("oem") + return stdout.decode(OEM) KEEPLIST = ( -- cgit v0.12